2024-11-15 21:54:33,958 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-15 21:54:33,977 main DEBUG Took 0.016503 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-15 21:54:33,977 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-15 21:54:33,977 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-15 21:54:33,978 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-15 21:54:33,979 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 21:54:33,986 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-15 21:54:33,999 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 21:54:34,001 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 21:54:34,002 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 21:54:34,002 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 21:54:34,003 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 21:54:34,003 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 21:54:34,005 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 21:54:34,005 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 21:54:34,006 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 21:54:34,006 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 21:54:34,008 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 21:54:34,008 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 21:54:34,009 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 21:54:34,009 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 21:54:34,010 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 21:54:34,010 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 21:54:34,011 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 21:54:34,011 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 21:54:34,012 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 21:54:34,012 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 21:54:34,013 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 21:54:34,013 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 21:54:34,014 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 21:54:34,015 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 21:54:34,015 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 21:54:34,016 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-15 21:54:34,017 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 21:54:34,019 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-15 21:54:34,021 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-15 21:54:34,022 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-15 21:54:34,024 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-15 21:54:34,024 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-15 21:54:34,035 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-15 21:54:34,039 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-15 21:54:34,041 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-15 21:54:34,041 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-15 21:54:34,042 main DEBUG createAppenders(={Console}) 2024-11-15 21:54:34,043 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-15 21:54:34,043 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-15 21:54:34,043 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-15 21:54:34,044 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-15 21:54:34,045 main DEBUG OutputStream closed 2024-11-15 21:54:34,045 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-15 21:54:34,045 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-15 21:54:34,046 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-15 21:54:34,137 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-15 21:54:34,139 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-15 21:54:34,141 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-15 21:54:34,142 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-15 21:54:34,142 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-15 21:54:34,143 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-15 21:54:34,143 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-15 21:54:34,144 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-15 21:54:34,144 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-15 21:54:34,145 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-15 21:54:34,145 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-15 21:54:34,146 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-15 21:54:34,146 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-15 21:54:34,147 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-15 21:54:34,147 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-15 21:54:34,147 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-15 21:54:34,148 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-15 21:54:34,149 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-15 21:54:34,152 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-15 21:54:34,152 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-15 21:54:34,153 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-15 21:54:34,153 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-15T21:54:34,173 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.procedure.TestSCPWithMeta timeout: 13 mins 2024-11-15 21:54:34,176 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-15 21:54:34,177 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-15T21:54:34,220 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.procedure.TestSCPWithMeta#testRecoveryAndDoubleExecutionOnRsWithMeta Thread=11, OpenFileDescriptor=285, MaxFileDescriptor=1048576, SystemLoadAverage=267, ProcessCount=11, AvailableMemoryMB=3628 2024-11-15T21:54:34,548 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ce7f090-fc0e-5e81-aacb-f56b26cecd72 2024-11-15T21:54:34,549 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-15T21:54:34,569 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ce7f090-fc0e-5e81-aacb-f56b26cecd72/cluster_370ef16d-d6cf-6544-0064-59de8e9843ef, deleteOnExit=true 2024-11-15T21:54:34,569 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-15T21:54:34,574 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ce7f090-fc0e-5e81-aacb-f56b26cecd72/test.cache.data in system properties and HBase conf 2024-11-15T21:54:34,575 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ce7f090-fc0e-5e81-aacb-f56b26cecd72/hadoop.tmp.dir in system properties and HBase conf 2024-11-15T21:54:34,575 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ce7f090-fc0e-5e81-aacb-f56b26cecd72/hadoop.log.dir in system properties and HBase conf 2024-11-15T21:54:34,576 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ce7f090-fc0e-5e81-aacb-f56b26cecd72/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-15T21:54:34,576 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ce7f090-fc0e-5e81-aacb-f56b26cecd72/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-15T21:54:34,577 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-15T21:54:34,768 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-15T21:54:34,890 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T21:54:34,896 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ce7f090-fc0e-5e81-aacb-f56b26cecd72/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-15T21:54:34,896 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ce7f090-fc0e-5e81-aacb-f56b26cecd72/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-15T21:54:34,897 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ce7f090-fc0e-5e81-aacb-f56b26cecd72/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-15T21:54:34,898 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ce7f090-fc0e-5e81-aacb-f56b26cecd72/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-15T21:54:34,899 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ce7f090-fc0e-5e81-aacb-f56b26cecd72/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-15T21:54:34,899 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ce7f090-fc0e-5e81-aacb-f56b26cecd72/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-15T21:54:34,900 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ce7f090-fc0e-5e81-aacb-f56b26cecd72/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-15T21:54:34,900 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ce7f090-fc0e-5e81-aacb-f56b26cecd72/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-15T21:54:34,901 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ce7f090-fc0e-5e81-aacb-f56b26cecd72/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-15T21:54:34,902 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ce7f090-fc0e-5e81-aacb-f56b26cecd72/nfs.dump.dir in system properties and HBase conf 2024-11-15T21:54:34,902 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ce7f090-fc0e-5e81-aacb-f56b26cecd72/java.io.tmpdir in system properties and HBase conf 2024-11-15T21:54:34,903 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ce7f090-fc0e-5e81-aacb-f56b26cecd72/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-15T21:54:34,903 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ce7f090-fc0e-5e81-aacb-f56b26cecd72/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-15T21:54:34,904 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ce7f090-fc0e-5e81-aacb-f56b26cecd72/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-15T21:54:35,783 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-15T21:54:35,895 INFO [Time-limited test {}] log.Log(170): Logging initialized @2676ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-15T21:54:36,001 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T21:54:36,093 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-15T21:54:36,130 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-15T21:54:36,130 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-15T21:54:36,131 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-15T21:54:36,144 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T21:54:36,147 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2c447bb0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ce7f090-fc0e-5e81-aacb-f56b26cecd72/hadoop.log.dir/,AVAILABLE} 2024-11-15T21:54:36,148 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4974766f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-15T21:54:36,367 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5183f126{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ce7f090-fc0e-5e81-aacb-f56b26cecd72/java.io.tmpdir/jetty-localhost-34689-hadoop-hdfs-3_4_1-tests_jar-_-any-11010092539967883896/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-15T21:54:36,375 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4350931a{HTTP/1.1, (http/1.1)}{localhost:34689} 2024-11-15T21:54:36,375 INFO [Time-limited test {}] server.Server(415): Started @3158ms 2024-11-15T21:54:36,800 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T21:54:36,807 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-15T21:54:36,809 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-15T21:54:36,809 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-15T21:54:36,809 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-15T21:54:36,810 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7da31712{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ce7f090-fc0e-5e81-aacb-f56b26cecd72/hadoop.log.dir/,AVAILABLE} 2024-11-15T21:54:36,811 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@664a7783{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-15T21:54:36,935 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4d81bf0e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ce7f090-fc0e-5e81-aacb-f56b26cecd72/java.io.tmpdir/jetty-localhost-43857-hadoop-hdfs-3_4_1-tests_jar-_-any-29522511849432632/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-15T21:54:36,935 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@572fe942{HTTP/1.1, (http/1.1)}{localhost:43857} 2024-11-15T21:54:36,936 INFO [Time-limited test {}] server.Server(415): Started @3718ms 2024-11-15T21:54:36,993 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-15T21:54:37,125 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T21:54:37,131 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-15T21:54:37,133 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-15T21:54:37,133 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-15T21:54:37,133 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-15T21:54:37,138 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7f4017e5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ce7f090-fc0e-5e81-aacb-f56b26cecd72/hadoop.log.dir/,AVAILABLE} 2024-11-15T21:54:37,138 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6c516a28{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-15T21:54:37,273 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1ba455be{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ce7f090-fc0e-5e81-aacb-f56b26cecd72/java.io.tmpdir/jetty-localhost-46467-hadoop-hdfs-3_4_1-tests_jar-_-any-1660099543354965919/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-15T21:54:37,274 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@259771a6{HTTP/1.1, (http/1.1)}{localhost:46467} 2024-11-15T21:54:37,274 INFO [Time-limited test {}] server.Server(415): Started @4057ms 2024-11-15T21:54:37,278 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-15T21:54:37,333 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T21:54:37,339 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-15T21:54:37,346 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-15T21:54:37,346 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-15T21:54:37,346 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-15T21:54:37,358 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@30067af9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ce7f090-fc0e-5e81-aacb-f56b26cecd72/hadoop.log.dir/,AVAILABLE} 2024-11-15T21:54:37,359 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@17888407{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-15T21:54:37,495 WARN [Thread-108 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ce7f090-fc0e-5e81-aacb-f56b26cecd72/cluster_370ef16d-d6cf-6544-0064-59de8e9843ef/data/data4/current/BP-1236864483-172.17.0.3-1731707675516/current, will proceed with Du for space computation calculation, 2024-11-15T21:54:37,495 WARN [Thread-107 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ce7f090-fc0e-5e81-aacb-f56b26cecd72/cluster_370ef16d-d6cf-6544-0064-59de8e9843ef/data/data2/current/BP-1236864483-172.17.0.3-1731707675516/current, will proceed with Du for space computation calculation, 2024-11-15T21:54:37,495 WARN [Thread-106 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ce7f090-fc0e-5e81-aacb-f56b26cecd72/cluster_370ef16d-d6cf-6544-0064-59de8e9843ef/data/data3/current/BP-1236864483-172.17.0.3-1731707675516/current, will proceed with Du for space computation calculation, 2024-11-15T21:54:37,498 WARN [Thread-105 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ce7f090-fc0e-5e81-aacb-f56b26cecd72/cluster_370ef16d-d6cf-6544-0064-59de8e9843ef/data/data1/current/BP-1236864483-172.17.0.3-1731707675516/current, will proceed with Du for space computation calculation, 2024-11-15T21:54:37,517 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@22333f25{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ce7f090-fc0e-5e81-aacb-f56b26cecd72/java.io.tmpdir/jetty-localhost-35049-hadoop-hdfs-3_4_1-tests_jar-_-any-12558681325459128761/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-15T21:54:37,518 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@eec57c1{HTTP/1.1, (http/1.1)}{localhost:35049} 2024-11-15T21:54:37,519 INFO [Time-limited test {}] server.Server(415): Started @4301ms 2024-11-15T21:54:37,522 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-15T21:54:37,570 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-15T21:54:37,571 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-15T21:54:37,652 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf87007fd6fc8213b with lease ID 0x9bf508b9d34213ae: Processing first storage report for DS-cfe332f0-c324-40d1-87f6-7313ac5c3921 from datanode DatanodeRegistration(127.0.0.1:40887, datanodeUuid=58e2f326-064e-4507-8270-f743dfbdcfa0, infoPort=44379, infoSecurePort=0, ipcPort=36335, storageInfo=lv=-57;cid=testClusterID;nsid=143663040;c=1731707675516) 2024-11-15T21:54:37,654 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf87007fd6fc8213b with lease ID 0x9bf508b9d34213ae: from storage DS-cfe332f0-c324-40d1-87f6-7313ac5c3921 node DatanodeRegistration(127.0.0.1:40887, datanodeUuid=58e2f326-064e-4507-8270-f743dfbdcfa0, infoPort=44379, infoSecurePort=0, ipcPort=36335, storageInfo=lv=-57;cid=testClusterID;nsid=143663040;c=1731707675516), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-15T21:54:37,655 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4cdc3ab77847b10c with lease ID 0x9bf508b9d34213af: Processing first storage report for DS-34745d98-d41f-404b-baa7-624040cb25c7 from datanode DatanodeRegistration(127.0.0.1:36461, datanodeUuid=949a27da-6531-4c55-b5ce-f3af4b331818, infoPort=41837, infoSecurePort=0, ipcPort=35591, storageInfo=lv=-57;cid=testClusterID;nsid=143663040;c=1731707675516) 2024-11-15T21:54:37,655 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4cdc3ab77847b10c with lease ID 0x9bf508b9d34213af: from storage DS-34745d98-d41f-404b-baa7-624040cb25c7 node DatanodeRegistration(127.0.0.1:36461, datanodeUuid=949a27da-6531-4c55-b5ce-f3af4b331818, infoPort=41837, infoSecurePort=0, ipcPort=35591, storageInfo=lv=-57;cid=testClusterID;nsid=143663040;c=1731707675516), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-15T21:54:37,655 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf87007fd6fc8213b with lease ID 0x9bf508b9d34213ae: Processing first storage report for DS-b9fa318c-a662-4fbf-ad79-097f3076ddb1 from datanode DatanodeRegistration(127.0.0.1:40887, datanodeUuid=58e2f326-064e-4507-8270-f743dfbdcfa0, infoPort=44379, infoSecurePort=0, ipcPort=36335, storageInfo=lv=-57;cid=testClusterID;nsid=143663040;c=1731707675516) 2024-11-15T21:54:37,655 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf87007fd6fc8213b with lease ID 0x9bf508b9d34213ae: from storage DS-b9fa318c-a662-4fbf-ad79-097f3076ddb1 node DatanodeRegistration(127.0.0.1:40887, datanodeUuid=58e2f326-064e-4507-8270-f743dfbdcfa0, infoPort=44379, infoSecurePort=0, ipcPort=36335, storageInfo=lv=-57;cid=testClusterID;nsid=143663040;c=1731707675516), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-15T21:54:37,656 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4cdc3ab77847b10c with lease ID 0x9bf508b9d34213af: Processing first storage report for DS-a86fe36d-3961-4d66-81d4-bf2c5673555d from datanode DatanodeRegistration(127.0.0.1:36461, datanodeUuid=949a27da-6531-4c55-b5ce-f3af4b331818, infoPort=41837, infoSecurePort=0, ipcPort=35591, storageInfo=lv=-57;cid=testClusterID;nsid=143663040;c=1731707675516) 2024-11-15T21:54:37,656 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4cdc3ab77847b10c with lease ID 0x9bf508b9d34213af: from storage DS-a86fe36d-3961-4d66-81d4-bf2c5673555d node DatanodeRegistration(127.0.0.1:36461, datanodeUuid=949a27da-6531-4c55-b5ce-f3af4b331818, infoPort=41837, infoSecurePort=0, ipcPort=35591, storageInfo=lv=-57;cid=testClusterID;nsid=143663040;c=1731707675516), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-15T21:54:37,686 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ce7f090-fc0e-5e81-aacb-f56b26cecd72/cluster_370ef16d-d6cf-6544-0064-59de8e9843ef/data/data6/current/BP-1236864483-172.17.0.3-1731707675516/current, will proceed with Du for space computation calculation, 2024-11-15T21:54:37,686 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ce7f090-fc0e-5e81-aacb-f56b26cecd72/cluster_370ef16d-d6cf-6544-0064-59de8e9843ef/data/data5/current/BP-1236864483-172.17.0.3-1731707675516/current, will proceed with Du for space computation calculation, 2024-11-15T21:54:37,720 WARN [Thread-123 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-15T21:54:37,726 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8edb86f7d2964e0e with lease ID 0x9bf508b9d34213b0: Processing first storage report for DS-f55f5b18-f6a1-4c42-97ca-ec4b858b4b31 from datanode DatanodeRegistration(127.0.0.1:41653, datanodeUuid=04ab36ac-20d1-483a-bb11-8fda764ca581, infoPort=46569, infoSecurePort=0, ipcPort=41321, storageInfo=lv=-57;cid=testClusterID;nsid=143663040;c=1731707675516) 2024-11-15T21:54:37,727 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8edb86f7d2964e0e with lease ID 0x9bf508b9d34213b0: from storage DS-f55f5b18-f6a1-4c42-97ca-ec4b858b4b31 node DatanodeRegistration(127.0.0.1:41653, datanodeUuid=04ab36ac-20d1-483a-bb11-8fda764ca581, infoPort=46569, infoSecurePort=0, ipcPort=41321, storageInfo=lv=-57;cid=testClusterID;nsid=143663040;c=1731707675516), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-15T21:54:37,727 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8edb86f7d2964e0e with lease ID 0x9bf508b9d34213b0: Processing first storage report for DS-65acb480-1352-40c3-8da1-3387e000c285 from datanode DatanodeRegistration(127.0.0.1:41653, datanodeUuid=04ab36ac-20d1-483a-bb11-8fda764ca581, infoPort=46569, infoSecurePort=0, ipcPort=41321, storageInfo=lv=-57;cid=testClusterID;nsid=143663040;c=1731707675516) 2024-11-15T21:54:37,727 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8edb86f7d2964e0e with lease ID 0x9bf508b9d34213b0: from storage DS-65acb480-1352-40c3-8da1-3387e000c285 node DatanodeRegistration(127.0.0.1:41653, datanodeUuid=04ab36ac-20d1-483a-bb11-8fda764ca581, infoPort=46569, infoSecurePort=0, ipcPort=41321, storageInfo=lv=-57;cid=testClusterID;nsid=143663040;c=1731707675516), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-15T21:54:37,964 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ce7f090-fc0e-5e81-aacb-f56b26cecd72 2024-11-15T21:54:38,069 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ce7f090-fc0e-5e81-aacb-f56b26cecd72/cluster_370ef16d-d6cf-6544-0064-59de8e9843ef/zookeeper_0, clientPort=58023, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ce7f090-fc0e-5e81-aacb-f56b26cecd72/cluster_370ef16d-d6cf-6544-0064-59de8e9843ef/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ce7f090-fc0e-5e81-aacb-f56b26cecd72/cluster_370ef16d-d6cf-6544-0064-59de8e9843ef/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-15T21:54:38,082 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58023 2024-11-15T21:54:38,097 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T21:54:38,100 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T21:54:38,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741825_1001 (size=7) 2024-11-15T21:54:38,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741825_1001 (size=7) 2024-11-15T21:54:38,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741825_1001 (size=7) 2024-11-15T21:54:38,433 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad with version=8 2024-11-15T21:54:38,433 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/hbase-staging 2024-11-15T21:54:38,565 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-15T21:54:38,864 INFO [Time-limited test {}] client.ConnectionUtils(128): master/ae3311ea5ccb:0 server-side Connection retries=45 2024-11-15T21:54:38,876 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T21:54:38,877 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T21:54:38,882 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T21:54:38,882 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T21:54:38,883 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T21:54:39,061 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-15T21:54:39,137 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-15T21:54:39,148 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-15T21:54:39,153 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T21:54:39,190 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 17311 (auto-detected) 2024-11-15T21:54:39,191 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-15T21:54:39,220 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:36531 2024-11-15T21:54:39,252 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36531 connecting to ZooKeeper ensemble=127.0.0.1:58023 2024-11-15T21:54:39,296 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:365310x0, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T21:54:39,305 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36531-0x100f86521fe0000 connected 2024-11-15T21:54:39,337 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T21:54:39,342 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T21:54:39,354 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36531-0x100f86521fe0000, quorum=127.0.0.1:58023, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T21:54:39,360 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad, hbase.cluster.distributed=false 2024-11-15T21:54:39,391 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36531-0x100f86521fe0000, quorum=127.0.0.1:58023, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T21:54:39,410 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36531 2024-11-15T21:54:39,411 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36531 2024-11-15T21:54:39,412 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36531 2024-11-15T21:54:39,417 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36531 2024-11-15T21:54:39,421 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36531 2024-11-15T21:54:39,565 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ae3311ea5ccb:0 server-side Connection retries=45 2024-11-15T21:54:39,567 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T21:54:39,567 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T21:54:39,567 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T21:54:39,567 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T21:54:39,568 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T21:54:39,570 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-15T21:54:39,573 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T21:54:39,574 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:40941 2024-11-15T21:54:39,576 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40941 connecting to ZooKeeper ensemble=127.0.0.1:58023 2024-11-15T21:54:39,578 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T21:54:39,583 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T21:54:39,592 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:409410x0, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T21:54:39,593 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:409410x0, quorum=127.0.0.1:58023, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T21:54:39,597 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40941-0x100f86521fe0001 connected 2024-11-15T21:54:39,598 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-15T21:54:39,607 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-15T21:54:39,610 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40941-0x100f86521fe0001, quorum=127.0.0.1:58023, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-15T21:54:39,619 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40941-0x100f86521fe0001, quorum=127.0.0.1:58023, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T21:54:39,619 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40941 2024-11-15T21:54:39,620 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40941 2024-11-15T21:54:39,620 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40941 2024-11-15T21:54:39,622 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40941 2024-11-15T21:54:39,622 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40941 2024-11-15T21:54:39,651 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ae3311ea5ccb:0 server-side Connection retries=45 2024-11-15T21:54:39,651 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T21:54:39,652 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T21:54:39,653 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T21:54:39,653 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T21:54:39,653 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T21:54:39,653 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-15T21:54:39,654 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T21:54:39,655 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:32871 2024-11-15T21:54:39,658 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:32871 connecting to ZooKeeper ensemble=127.0.0.1:58023 2024-11-15T21:54:39,659 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T21:54:39,664 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T21:54:39,672 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:328710x0, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T21:54:39,672 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:32871-0x100f86521fe0002 connected 2024-11-15T21:54:39,677 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32871-0x100f86521fe0002, quorum=127.0.0.1:58023, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T21:54:39,677 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-15T21:54:39,682 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-15T21:54:39,684 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32871-0x100f86521fe0002, quorum=127.0.0.1:58023, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-15T21:54:39,686 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32871-0x100f86521fe0002, quorum=127.0.0.1:58023, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T21:54:39,687 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=32871 2024-11-15T21:54:39,689 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=32871 2024-11-15T21:54:39,692 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=32871 2024-11-15T21:54:39,697 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=32871 2024-11-15T21:54:39,698 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=32871 2024-11-15T21:54:39,726 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ae3311ea5ccb:0 server-side Connection retries=45 2024-11-15T21:54:39,726 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T21:54:39,727 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T21:54:39,727 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T21:54:39,727 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T21:54:39,727 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T21:54:39,727 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-15T21:54:39,728 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T21:54:39,729 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:37455 2024-11-15T21:54:39,731 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37455 connecting to ZooKeeper ensemble=127.0.0.1:58023 2024-11-15T21:54:39,733 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T21:54:39,736 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T21:54:39,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:374550x0, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T21:54:39,750 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:374550x0, quorum=127.0.0.1:58023, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T21:54:39,750 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37455-0x100f86521fe0003 connected 2024-11-15T21:54:39,750 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-15T21:54:39,756 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-15T21:54:39,758 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37455-0x100f86521fe0003, quorum=127.0.0.1:58023, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-15T21:54:39,761 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37455-0x100f86521fe0003, quorum=127.0.0.1:58023, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T21:54:39,765 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37455 2024-11-15T21:54:39,766 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37455 2024-11-15T21:54:39,767 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37455 2024-11-15T21:54:39,768 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37455 2024-11-15T21:54:39,769 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37455 2024-11-15T21:54:39,785 DEBUG [M:0;ae3311ea5ccb:36531 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;ae3311ea5ccb:36531 2024-11-15T21:54:39,786 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/ae3311ea5ccb,36531,1731707678636 2024-11-15T21:54:39,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37455-0x100f86521fe0003, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T21:54:39,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40941-0x100f86521fe0001, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T21:54:39,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36531-0x100f86521fe0000, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T21:54:39,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32871-0x100f86521fe0002, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T21:54:39,795 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36531-0x100f86521fe0000, quorum=127.0.0.1:58023, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/ae3311ea5ccb,36531,1731707678636 2024-11-15T21:54:39,825 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37455-0x100f86521fe0003, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-15T21:54:39,825 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40941-0x100f86521fe0001, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-15T21:54:39,825 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36531-0x100f86521fe0000, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:54:39,825 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32871-0x100f86521fe0002, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-15T21:54:39,825 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37455-0x100f86521fe0003, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:54:39,825 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40941-0x100f86521fe0001, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:54:39,825 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32871-0x100f86521fe0002, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:54:39,827 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36531-0x100f86521fe0000, quorum=127.0.0.1:58023, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-15T21:54:39,829 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/ae3311ea5ccb,36531,1731707678636 from backup master directory 2024-11-15T21:54:39,832 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40941-0x100f86521fe0001, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T21:54:39,832 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37455-0x100f86521fe0003, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T21:54:39,832 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32871-0x100f86521fe0002, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T21:54:39,832 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36531-0x100f86521fe0000, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/ae3311ea5ccb,36531,1731707678636 2024-11-15T21:54:39,832 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36531-0x100f86521fe0000, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T21:54:39,833 WARN [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T21:54:39,834 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=ae3311ea5ccb,36531,1731707678636 2024-11-15T21:54:39,836 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-15T21:54:39,837 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-15T21:54:39,898 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/hbase.id] with ID: 4680bf58-6f3b-4f68-8171-8cc0087dc31c 2024-11-15T21:54:39,898 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/.tmp/hbase.id 2024-11-15T21:54:39,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741826_1002 (size=42) 2024-11-15T21:54:39,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741826_1002 (size=42) 2024-11-15T21:54:39,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741826_1002 (size=42) 2024-11-15T21:54:39,916 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/.tmp/hbase.id]:[hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/hbase.id] 2024-11-15T21:54:39,960 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T21:54:39,966 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-15T21:54:39,989 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 21ms. 2024-11-15T21:54:39,993 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37455-0x100f86521fe0003, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:54:39,993 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40941-0x100f86521fe0001, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:54:39,993 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36531-0x100f86521fe0000, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:54:39,993 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32871-0x100f86521fe0002, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:54:40,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741827_1003 (size=196) 2024-11-15T21:54:40,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741827_1003 (size=196) 2024-11-15T21:54:40,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741827_1003 (size=196) 2024-11-15T21:54:40,036 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T21:54:40,038 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-15T21:54:40,055 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T21:54:40,061 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T21:54:40,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741828_1004 (size=1189) 2024-11-15T21:54:40,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741828_1004 (size=1189) 2024-11-15T21:54:40,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741828_1004 (size=1189) 2024-11-15T21:54:40,127 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/MasterData/data/master/store 2024-11-15T21:54:40,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741829_1005 (size=34) 2024-11-15T21:54:40,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741829_1005 (size=34) 2024-11-15T21:54:40,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741829_1005 (size=34) 2024-11-15T21:54:40,167 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-15T21:54:40,170 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:40,172 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-15T21:54:40,172 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T21:54:40,172 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T21:54:40,175 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-15T21:54:40,175 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T21:54:40,175 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T21:54:40,177 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731707680171Disabling compacts and flushes for region at 1731707680171Disabling writes for close at 1731707680175 (+4 ms)Writing region close event to WAL at 1731707680175Closed at 1731707680175 2024-11-15T21:54:40,180 WARN [master/ae3311ea5ccb:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/MasterData/data/master/store/.initializing 2024-11-15T21:54:40,181 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/MasterData/WALs/ae3311ea5ccb,36531,1731707678636 2024-11-15T21:54:40,191 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T21:54:40,211 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ae3311ea5ccb%2C36531%2C1731707678636, suffix=, logDir=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/MasterData/WALs/ae3311ea5ccb,36531,1731707678636, archiveDir=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/MasterData/oldWALs, maxLogs=10 2024-11-15T21:54:40,238 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/MasterData/WALs/ae3311ea5ccb,36531,1731707678636/ae3311ea5ccb%2C36531%2C1731707678636.1731707680217, exclude list is [], retry=0 2024-11-15T21:54:40,262 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40887,DS-cfe332f0-c324-40d1-87f6-7313ac5c3921,DISK] 2024-11-15T21:54:40,262 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41653,DS-f55f5b18-f6a1-4c42-97ca-ec4b858b4b31,DISK] 2024-11-15T21:54:40,262 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36461,DS-34745d98-d41f-404b-baa7-624040cb25c7,DISK] 2024-11-15T21:54:40,265 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-15T21:54:40,312 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/MasterData/WALs/ae3311ea5ccb,36531,1731707678636/ae3311ea5ccb%2C36531%2C1731707678636.1731707680217 2024-11-15T21:54:40,313 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46569:46569),(127.0.0.1/127.0.0.1:41837:41837),(127.0.0.1/127.0.0.1:44379:44379)] 2024-11-15T21:54:40,314 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-15T21:54:40,314 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:40,318 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T21:54:40,319 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T21:54:40,368 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T21:54:40,403 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-15T21:54:40,407 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:40,411 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T21:54:40,412 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T21:54:40,418 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-15T21:54:40,418 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:40,419 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:40,420 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T21:54:40,423 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-15T21:54:40,423 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:40,424 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:40,424 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T21:54:40,427 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-15T21:54:40,428 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:40,428 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:40,429 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T21:54:40,437 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-15T21:54:40,439 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-15T21:54:40,445 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T21:54:40,446 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T21:54:40,449 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-15T21:54:40,453 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T21:54:40,459 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T21:54:40,460 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59129394, jitterRate=-0.11890336871147156}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-15T21:54:40,469 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731707680332Initializing all the Stores at 1731707680335 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731707680336 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707680337 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707680337Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707680337Cleaning up temporary data from old regions at 1731707680446 (+109 ms)Region opened successfully at 1731707680469 (+23 ms) 2024-11-15T21:54:40,470 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-15T21:54:40,525 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54ab0222, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ae3311ea5ccb/172.17.0.3:0 2024-11-15T21:54:40,559 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-15T21:54:40,573 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T21:54:40,573 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T21:54:40,576 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T21:54:40,578 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-15T21:54:40,585 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-15T21:54:40,585 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-15T21:54:40,615 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-15T21:54:40,624 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36531-0x100f86521fe0000, quorum=127.0.0.1:58023, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-15T21:54:40,626 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-15T21:54:40,629 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-15T21:54:40,632 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36531-0x100f86521fe0000, quorum=127.0.0.1:58023, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-15T21:54:40,634 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-15T21:54:40,636 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-15T21:54:40,640 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36531-0x100f86521fe0000, quorum=127.0.0.1:58023, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-15T21:54:40,643 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-15T21:54:40,644 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36531-0x100f86521fe0000, quorum=127.0.0.1:58023, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-15T21:54:40,646 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-15T21:54:40,670 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36531-0x100f86521fe0000, quorum=127.0.0.1:58023, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-15T21:54:40,672 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-15T21:54:40,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40941-0x100f86521fe0001, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-15T21:54:40,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40941-0x100f86521fe0001, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:54:40,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32871-0x100f86521fe0002, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-15T21:54:40,679 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32871-0x100f86521fe0002, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:54:40,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37455-0x100f86521fe0003, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-15T21:54:40,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37455-0x100f86521fe0003, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:54:40,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36531-0x100f86521fe0000, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-15T21:54:40,682 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36531-0x100f86521fe0000, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:54:40,688 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=ae3311ea5ccb,36531,1731707678636, sessionid=0x100f86521fe0000, setting cluster-up flag (Was=false) 2024-11-15T21:54:40,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40941-0x100f86521fe0001, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:54:40,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32871-0x100f86521fe0002, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:54:40,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37455-0x100f86521fe0003, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:54:40,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36531-0x100f86521fe0000, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:54:40,719 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-15T21:54:40,721 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ae3311ea5ccb,36531,1731707678636 2024-11-15T21:54:40,727 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37455-0x100f86521fe0003, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:54:40,727 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36531-0x100f86521fe0000, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:54:40,728 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32871-0x100f86521fe0002, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:54:40,727 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40941-0x100f86521fe0001, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:54:40,736 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-15T21:54:40,738 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ae3311ea5ccb,36531,1731707678636 2024-11-15T21:54:40,746 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-15T21:54:40,774 INFO [RS:0;ae3311ea5ccb:40941 {}] regionserver.HRegionServer(746): ClusterId : 4680bf58-6f3b-4f68-8171-8cc0087dc31c 2024-11-15T21:54:40,774 INFO [RS:2;ae3311ea5ccb:37455 {}] regionserver.HRegionServer(746): ClusterId : 4680bf58-6f3b-4f68-8171-8cc0087dc31c 2024-11-15T21:54:40,778 DEBUG [RS:0;ae3311ea5ccb:40941 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-15T21:54:40,778 DEBUG [RS:2;ae3311ea5ccb:37455 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-15T21:54:40,778 INFO [RS:1;ae3311ea5ccb:32871 {}] regionserver.HRegionServer(746): ClusterId : 4680bf58-6f3b-4f68-8171-8cc0087dc31c 2024-11-15T21:54:40,779 DEBUG [RS:1;ae3311ea5ccb:32871 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-15T21:54:40,785 DEBUG [RS:0;ae3311ea5ccb:40941 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-15T21:54:40,785 DEBUG [RS:0;ae3311ea5ccb:40941 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-15T21:54:40,790 DEBUG [RS:2;ae3311ea5ccb:37455 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-15T21:54:40,790 DEBUG [RS:2;ae3311ea5ccb:37455 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-15T21:54:40,790 DEBUG [RS:1;ae3311ea5ccb:32871 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-15T21:54:40,790 DEBUG [RS:1;ae3311ea5ccb:32871 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-15T21:54:40,794 DEBUG [RS:2;ae3311ea5ccb:37455 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-15T21:54:40,794 DEBUG [RS:0;ae3311ea5ccb:40941 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-15T21:54:40,794 DEBUG [RS:1;ae3311ea5ccb:32871 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-15T21:54:40,795 DEBUG [RS:2;ae3311ea5ccb:37455 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b92d6dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ae3311ea5ccb/172.17.0.3:0 2024-11-15T21:54:40,795 DEBUG [RS:0;ae3311ea5ccb:40941 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1083e00b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ae3311ea5ccb/172.17.0.3:0 2024-11-15T21:54:40,795 DEBUG [RS:1;ae3311ea5ccb:32871 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cee2486, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ae3311ea5ccb/172.17.0.3:0 2024-11-15T21:54:40,818 DEBUG [RS:1;ae3311ea5ccb:32871 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;ae3311ea5ccb:32871 2024-11-15T21:54:40,818 DEBUG [RS:2;ae3311ea5ccb:37455 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;ae3311ea5ccb:37455 2024-11-15T21:54:40,818 DEBUG [RS:0;ae3311ea5ccb:40941 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;ae3311ea5ccb:40941 2024-11-15T21:54:40,822 INFO [RS:1;ae3311ea5ccb:32871 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-15T21:54:40,822 INFO [RS:0;ae3311ea5ccb:40941 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-15T21:54:40,822 INFO [RS:2;ae3311ea5ccb:37455 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-15T21:54:40,822 INFO [RS:1;ae3311ea5ccb:32871 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-15T21:54:40,822 INFO [RS:2;ae3311ea5ccb:37455 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-15T21:54:40,822 INFO [RS:0;ae3311ea5ccb:40941 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-15T21:54:40,822 DEBUG [RS:2;ae3311ea5ccb:37455 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-15T21:54:40,822 DEBUG [RS:1;ae3311ea5ccb:32871 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-15T21:54:40,822 DEBUG [RS:0;ae3311ea5ccb:40941 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-15T21:54:40,825 INFO [RS:1;ae3311ea5ccb:32871 {}] regionserver.HRegionServer(2659): reportForDuty to master=ae3311ea5ccb,36531,1731707678636 with port=32871, startcode=1731707679650 2024-11-15T21:54:40,825 INFO [RS:2;ae3311ea5ccb:37455 {}] regionserver.HRegionServer(2659): reportForDuty to master=ae3311ea5ccb,36531,1731707678636 with port=37455, startcode=1731707679725 2024-11-15T21:54:40,825 INFO [RS:0;ae3311ea5ccb:40941 {}] regionserver.HRegionServer(2659): reportForDuty to master=ae3311ea5ccb,36531,1731707678636 with port=40941, startcode=1731707679516 2024-11-15T21:54:40,842 DEBUG [RS:1;ae3311ea5ccb:32871 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-15T21:54:40,845 DEBUG [RS:0;ae3311ea5ccb:40941 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-15T21:54:40,845 DEBUG [RS:2;ae3311ea5ccb:37455 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-15T21:54:40,859 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T21:54:40,877 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-15T21:54:40,892 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56291, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-15T21:54:40,892 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34467, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-15T21:54:40,893 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49809, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-15T21:54:40,896 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-15T21:54:40,903 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36531 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-15T21:54:40,910 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36531 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-15T21:54:40,911 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36531 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-15T21:54:40,909 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: ae3311ea5ccb,36531,1731707678636 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-15T21:54:40,924 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/ae3311ea5ccb:0, corePoolSize=5, maxPoolSize=5 2024-11-15T21:54:40,924 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/ae3311ea5ccb:0, corePoolSize=5, maxPoolSize=5 2024-11-15T21:54:40,925 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/ae3311ea5ccb:0, corePoolSize=5, maxPoolSize=5 2024-11-15T21:54:40,925 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/ae3311ea5ccb:0, corePoolSize=5, maxPoolSize=5 2024-11-15T21:54:40,925 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/ae3311ea5ccb:0, corePoolSize=10, maxPoolSize=10 2024-11-15T21:54:40,925 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:54:40,925 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/ae3311ea5ccb:0, corePoolSize=2, maxPoolSize=2 2024-11-15T21:54:40,926 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:54:40,930 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731707710929 2024-11-15T21:54:40,932 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-15T21:54:40,933 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-15T21:54:40,936 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T21:54:40,936 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-15T21:54:40,937 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-15T21:54:40,937 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-15T21:54:40,937 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-15T21:54:40,937 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-15T21:54:40,940 DEBUG [RS:0;ae3311ea5ccb:40941 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-15T21:54:40,940 DEBUG [RS:1;ae3311ea5ccb:32871 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-15T21:54:40,940 WARN [RS:0;ae3311ea5ccb:40941 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-15T21:54:40,940 DEBUG [RS:2;ae3311ea5ccb:37455 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-15T21:54:40,940 WARN [RS:2;ae3311ea5ccb:37455 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-15T21:54:40,940 WARN [RS:1;ae3311ea5ccb:32871 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-15T21:54:40,940 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-15T21:54:40,942 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-15T21:54:40,943 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-15T21:54:40,943 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-15T21:54:40,945 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-15T21:54:40,946 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-15T21:54:40,948 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:40,948 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/ae3311ea5ccb:0:becomeActiveMaster-HFileCleaner.large.0-1731707680947,5,FailOnTimeoutGroup] 2024-11-15T21:54:40,948 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-15T21:54:40,952 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/ae3311ea5ccb:0:becomeActiveMaster-HFileCleaner.small.0-1731707680948,5,FailOnTimeoutGroup] 2024-11-15T21:54:40,952 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-15T21:54:40,952 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-15T21:54:40,954 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-15T21:54:40,954 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-15T21:54:40,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741831_1007 (size=1321) 2024-11-15T21:54:40,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741831_1007 (size=1321) 2024-11-15T21:54:40,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741831_1007 (size=1321) 2024-11-15T21:54:41,003 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-15T21:54:41,003 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad 2024-11-15T21:54:41,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741832_1008 (size=32) 2024-11-15T21:54:41,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741832_1008 (size=32) 2024-11-15T21:54:41,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741832_1008 (size=32) 2024-11-15T21:54:41,027 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:41,029 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-15T21:54:41,032 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-15T21:54:41,032 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:41,033 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T21:54:41,034 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-15T21:54:41,037 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-15T21:54:41,037 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:41,038 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T21:54:41,038 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-15T21:54:41,041 INFO [RS:0;ae3311ea5ccb:40941 {}] regionserver.HRegionServer(2659): reportForDuty to master=ae3311ea5ccb,36531,1731707678636 with port=40941, startcode=1731707679516 2024-11-15T21:54:41,041 INFO [RS:2;ae3311ea5ccb:37455 {}] regionserver.HRegionServer(2659): reportForDuty to master=ae3311ea5ccb,36531,1731707678636 with port=37455, startcode=1731707679725 2024-11-15T21:54:41,041 INFO [RS:1;ae3311ea5ccb:32871 {}] regionserver.HRegionServer(2659): reportForDuty to master=ae3311ea5ccb,36531,1731707678636 with port=32871, startcode=1731707679650 2024-11-15T21:54:41,043 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-15T21:54:41,043 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:41,044 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36531 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ae3311ea5ccb,40941,1731707679516 2024-11-15T21:54:41,044 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T21:54:41,045 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-15T21:54:41,047 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36531 {}] master.ServerManager(517): Registering regionserver=ae3311ea5ccb,40941,1731707679516 2024-11-15T21:54:41,062 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-15T21:54:41,062 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:41,064 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T21:54:41,064 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-15T21:54:41,066 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/hbase/meta/1588230740 2024-11-15T21:54:41,067 DEBUG [RS:0;ae3311ea5ccb:40941 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad 2024-11-15T21:54:41,067 DEBUG [RS:0;ae3311ea5ccb:40941 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45565 2024-11-15T21:54:41,067 DEBUG [RS:0;ae3311ea5ccb:40941 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-15T21:54:41,067 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/hbase/meta/1588230740 2024-11-15T21:54:41,067 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36531 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ae3311ea5ccb,37455,1731707679725 2024-11-15T21:54:41,068 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36531 {}] master.ServerManager(517): Registering regionserver=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:54:41,073 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-15T21:54:41,073 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-15T21:54:41,074 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36531-0x100f86521fe0000, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T21:54:41,074 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36531 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ae3311ea5ccb,32871,1731707679650 2024-11-15T21:54:41,074 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36531 {}] master.ServerManager(517): Registering regionserver=ae3311ea5ccb,32871,1731707679650 2024-11-15T21:54:41,075 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-15T21:54:41,075 DEBUG [RS:2;ae3311ea5ccb:37455 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad 2024-11-15T21:54:41,075 DEBUG [RS:2;ae3311ea5ccb:37455 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45565 2024-11-15T21:54:41,075 DEBUG [RS:0;ae3311ea5ccb:40941 {}] zookeeper.ZKUtil(111): regionserver:40941-0x100f86521fe0001, quorum=127.0.0.1:58023, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ae3311ea5ccb,40941,1731707679516 2024-11-15T21:54:41,075 DEBUG [RS:2;ae3311ea5ccb:37455 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-15T21:54:41,075 WARN [RS:0;ae3311ea5ccb:40941 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T21:54:41,075 INFO [RS:0;ae3311ea5ccb:40941 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T21:54:41,076 DEBUG [RS:0;ae3311ea5ccb:40941 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/WALs/ae3311ea5ccb,40941,1731707679516 2024-11-15T21:54:41,078 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-15T21:54:41,084 DEBUG [RS:2;ae3311ea5ccb:37455 {}] zookeeper.ZKUtil(111): regionserver:37455-0x100f86521fe0003, quorum=127.0.0.1:58023, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ae3311ea5ccb,37455,1731707679725 2024-11-15T21:54:41,084 WARN [RS:2;ae3311ea5ccb:37455 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T21:54:41,084 INFO [RS:2;ae3311ea5ccb:37455 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T21:54:41,084 DEBUG [RS:2;ae3311ea5ccb:37455 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/WALs/ae3311ea5ccb,37455,1731707679725 2024-11-15T21:54:41,086 DEBUG [RS:1;ae3311ea5ccb:32871 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad 2024-11-15T21:54:41,086 DEBUG [RS:1;ae3311ea5ccb:32871 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45565 2024-11-15T21:54:41,087 DEBUG [RS:1;ae3311ea5ccb:32871 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-15T21:54:41,087 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ae3311ea5ccb,37455,1731707679725] 2024-11-15T21:54:41,087 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ae3311ea5ccb,40941,1731707679516] 2024-11-15T21:54:41,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36531-0x100f86521fe0000, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T21:54:41,092 DEBUG [RS:1;ae3311ea5ccb:32871 {}] zookeeper.ZKUtil(111): regionserver:32871-0x100f86521fe0002, quorum=127.0.0.1:58023, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ae3311ea5ccb,32871,1731707679650 2024-11-15T21:54:41,092 WARN [RS:1;ae3311ea5ccb:32871 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T21:54:41,093 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ae3311ea5ccb,32871,1731707679650] 2024-11-15T21:54:41,093 INFO [RS:1;ae3311ea5ccb:32871 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T21:54:41,093 DEBUG [RS:1;ae3311ea5ccb:32871 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/WALs/ae3311ea5ccb,32871,1731707679650 2024-11-15T21:54:41,101 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T21:54:41,103 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62812113, jitterRate=-0.06402657926082611}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-15T21:54:41,106 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731707681027Initializing all the Stores at 1731707681029 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731707681029Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731707681029Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707681029Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731707681029Cleaning up temporary data from old regions at 1731707681073 (+44 ms)Region opened successfully at 1731707681106 (+33 ms) 2024-11-15T21:54:41,106 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-15T21:54:41,107 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-15T21:54:41,107 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-15T21:54:41,107 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-15T21:54:41,107 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-15T21:54:41,110 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-15T21:54:41,110 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731707681106Disabling compacts and flushes for region at 1731707681106Disabling writes for close at 1731707681107 (+1 ms)Writing region close event to WAL at 1731707681110 (+3 ms)Closed at 1731707681110 2024-11-15T21:54:41,122 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T21:54:41,122 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-15T21:54:41,128 INFO [RS:0;ae3311ea5ccb:40941 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-15T21:54:41,128 INFO [RS:2;ae3311ea5ccb:37455 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-15T21:54:41,128 INFO [RS:1;ae3311ea5ccb:32871 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-15T21:54:41,129 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-15T21:54:41,152 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T21:54:41,159 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-15T21:54:41,168 INFO [RS:2;ae3311ea5ccb:37455 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-15T21:54:41,168 INFO [RS:0;ae3311ea5ccb:40941 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-15T21:54:41,169 INFO [RS:1;ae3311ea5ccb:32871 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-15T21:54:41,182 INFO [RS:1;ae3311ea5ccb:32871 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-15T21:54:41,182 INFO [RS:0;ae3311ea5ccb:40941 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-15T21:54:41,182 INFO [RS:1;ae3311ea5ccb:32871 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T21:54:41,182 INFO [RS:0;ae3311ea5ccb:40941 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T21:54:41,184 INFO [RS:2;ae3311ea5ccb:37455 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-15T21:54:41,185 INFO [RS:2;ae3311ea5ccb:37455 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T21:54:41,186 INFO [RS:2;ae3311ea5ccb:37455 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-15T21:54:41,189 INFO [RS:0;ae3311ea5ccb:40941 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-15T21:54:41,189 INFO [RS:1;ae3311ea5ccb:32871 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-15T21:54:41,194 INFO [RS:1;ae3311ea5ccb:32871 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-15T21:54:41,197 INFO [RS:2;ae3311ea5ccb:37455 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-15T21:54:41,195 INFO [RS:0;ae3311ea5ccb:40941 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-15T21:54:41,199 INFO [RS:1;ae3311ea5ccb:32871 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-15T21:54:41,199 DEBUG [RS:1;ae3311ea5ccb:32871 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:54:41,199 DEBUG [RS:1;ae3311ea5ccb:32871 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:54:41,199 DEBUG [RS:1;ae3311ea5ccb:32871 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:54:41,199 DEBUG [RS:1;ae3311ea5ccb:32871 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:54:41,200 DEBUG [RS:1;ae3311ea5ccb:32871 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:54:41,200 INFO [RS:2;ae3311ea5ccb:37455 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-15T21:54:41,200 DEBUG [RS:1;ae3311ea5ccb:32871 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0, corePoolSize=2, maxPoolSize=2 2024-11-15T21:54:41,200 DEBUG [RS:2;ae3311ea5ccb:37455 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:54:41,200 DEBUG [RS:2;ae3311ea5ccb:37455 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:54:41,200 DEBUG [RS:2;ae3311ea5ccb:37455 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:54:41,200 DEBUG [RS:2;ae3311ea5ccb:37455 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:54:41,200 DEBUG [RS:2;ae3311ea5ccb:37455 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:54:41,201 DEBUG [RS:2;ae3311ea5ccb:37455 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0, corePoolSize=2, maxPoolSize=2 2024-11-15T21:54:41,201 DEBUG [RS:2;ae3311ea5ccb:37455 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:54:41,200 DEBUG [RS:1;ae3311ea5ccb:32871 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:54:41,201 INFO [RS:0;ae3311ea5ccb:40941 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-15T21:54:41,201 DEBUG [RS:1;ae3311ea5ccb:32871 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:54:41,201 DEBUG [RS:0;ae3311ea5ccb:40941 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:54:41,201 DEBUG [RS:2;ae3311ea5ccb:37455 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:54:41,201 DEBUG [RS:1;ae3311ea5ccb:32871 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:54:41,201 DEBUG [RS:1;ae3311ea5ccb:32871 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:54:41,201 DEBUG [RS:2;ae3311ea5ccb:37455 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:54:41,201 DEBUG [RS:1;ae3311ea5ccb:32871 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:54:41,202 DEBUG [RS:2;ae3311ea5ccb:37455 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:54:41,202 DEBUG [RS:1;ae3311ea5ccb:32871 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:54:41,202 DEBUG [RS:2;ae3311ea5ccb:37455 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:54:41,202 DEBUG [RS:1;ae3311ea5ccb:32871 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ae3311ea5ccb:0, corePoolSize=3, maxPoolSize=3 2024-11-15T21:54:41,202 DEBUG [RS:1;ae3311ea5ccb:32871 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0, corePoolSize=3, maxPoolSize=3 2024-11-15T21:54:41,202 DEBUG [RS:2;ae3311ea5ccb:37455 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:54:41,202 DEBUG [RS:2;ae3311ea5ccb:37455 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ae3311ea5ccb:0, corePoolSize=3, maxPoolSize=3 2024-11-15T21:54:41,202 DEBUG [RS:2;ae3311ea5ccb:37455 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0, corePoolSize=3, maxPoolSize=3 2024-11-15T21:54:41,202 DEBUG [RS:0;ae3311ea5ccb:40941 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:54:41,202 DEBUG [RS:0;ae3311ea5ccb:40941 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:54:41,202 DEBUG [RS:0;ae3311ea5ccb:40941 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:54:41,203 DEBUG [RS:0;ae3311ea5ccb:40941 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:54:41,203 DEBUG [RS:0;ae3311ea5ccb:40941 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0, corePoolSize=2, maxPoolSize=2 2024-11-15T21:54:41,203 DEBUG [RS:0;ae3311ea5ccb:40941 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:54:41,203 DEBUG [RS:0;ae3311ea5ccb:40941 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:54:41,203 DEBUG [RS:0;ae3311ea5ccb:40941 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:54:41,203 DEBUG [RS:0;ae3311ea5ccb:40941 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:54:41,203 DEBUG [RS:0;ae3311ea5ccb:40941 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:54:41,203 DEBUG [RS:0;ae3311ea5ccb:40941 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:54:41,203 DEBUG [RS:0;ae3311ea5ccb:40941 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ae3311ea5ccb:0, corePoolSize=3, maxPoolSize=3 2024-11-15T21:54:41,203 DEBUG [RS:0;ae3311ea5ccb:40941 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0, corePoolSize=3, maxPoolSize=3 2024-11-15T21:54:41,206 INFO [RS:1;ae3311ea5ccb:32871 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T21:54:41,207 INFO [RS:1;ae3311ea5ccb:32871 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T21:54:41,207 INFO [RS:1;ae3311ea5ccb:32871 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T21:54:41,207 INFO [RS:1;ae3311ea5ccb:32871 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-15T21:54:41,207 INFO [RS:1;ae3311ea5ccb:32871 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-15T21:54:41,207 INFO [RS:1;ae3311ea5ccb:32871 {}] hbase.ChoreService(168): Chore ScheduledChore name=ae3311ea5ccb,32871,1731707679650-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T21:54:41,209 INFO [RS:2;ae3311ea5ccb:37455 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T21:54:41,209 INFO [RS:2;ae3311ea5ccb:37455 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T21:54:41,209 INFO [RS:2;ae3311ea5ccb:37455 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T21:54:41,209 INFO [RS:2;ae3311ea5ccb:37455 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-15T21:54:41,210 INFO [RS:2;ae3311ea5ccb:37455 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-15T21:54:41,210 INFO [RS:2;ae3311ea5ccb:37455 {}] hbase.ChoreService(168): Chore ScheduledChore name=ae3311ea5ccb,37455,1731707679725-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T21:54:41,215 INFO [RS:0;ae3311ea5ccb:40941 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T21:54:41,215 INFO [RS:0;ae3311ea5ccb:40941 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T21:54:41,215 INFO [RS:0;ae3311ea5ccb:40941 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T21:54:41,215 INFO [RS:0;ae3311ea5ccb:40941 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-15T21:54:41,215 INFO [RS:0;ae3311ea5ccb:40941 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-15T21:54:41,215 INFO [RS:0;ae3311ea5ccb:40941 {}] hbase.ChoreService(168): Chore ScheduledChore name=ae3311ea5ccb,40941,1731707679516-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T21:54:41,234 INFO [RS:2;ae3311ea5ccb:37455 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-15T21:54:41,236 INFO [RS:1;ae3311ea5ccb:32871 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-15T21:54:41,237 INFO [RS:2;ae3311ea5ccb:37455 {}] hbase.ChoreService(168): Chore ScheduledChore name=ae3311ea5ccb,37455,1731707679725-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T21:54:41,237 INFO [RS:1;ae3311ea5ccb:32871 {}] hbase.ChoreService(168): Chore ScheduledChore name=ae3311ea5ccb,32871,1731707679650-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T21:54:41,237 INFO [RS:2;ae3311ea5ccb:37455 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T21:54:41,237 INFO [RS:2;ae3311ea5ccb:37455 {}] regionserver.Replication(171): ae3311ea5ccb,37455,1731707679725 started 2024-11-15T21:54:41,238 INFO [RS:1;ae3311ea5ccb:32871 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T21:54:41,238 INFO [RS:1;ae3311ea5ccb:32871 {}] regionserver.Replication(171): ae3311ea5ccb,32871,1731707679650 started 2024-11-15T21:54:41,240 INFO [RS:0;ae3311ea5ccb:40941 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-15T21:54:41,240 INFO [RS:0;ae3311ea5ccb:40941 {}] hbase.ChoreService(168): Chore ScheduledChore name=ae3311ea5ccb,40941,1731707679516-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T21:54:41,241 INFO [RS:0;ae3311ea5ccb:40941 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T21:54:41,241 INFO [RS:0;ae3311ea5ccb:40941 {}] regionserver.Replication(171): ae3311ea5ccb,40941,1731707679516 started 2024-11-15T21:54:41,261 INFO [RS:0;ae3311ea5ccb:40941 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T21:54:41,261 INFO [RS:0;ae3311ea5ccb:40941 {}] regionserver.HRegionServer(1482): Serving as ae3311ea5ccb,40941,1731707679516, RpcServer on ae3311ea5ccb/172.17.0.3:40941, sessionid=0x100f86521fe0001 2024-11-15T21:54:41,262 DEBUG [RS:0;ae3311ea5ccb:40941 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-15T21:54:41,262 DEBUG [RS:0;ae3311ea5ccb:40941 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ae3311ea5ccb,40941,1731707679516 2024-11-15T21:54:41,263 DEBUG [RS:0;ae3311ea5ccb:40941 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ae3311ea5ccb,40941,1731707679516' 2024-11-15T21:54:41,263 DEBUG [RS:0;ae3311ea5ccb:40941 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-15T21:54:41,264 INFO [RS:2;ae3311ea5ccb:37455 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T21:54:41,264 INFO [RS:1;ae3311ea5ccb:32871 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T21:54:41,264 INFO [RS:2;ae3311ea5ccb:37455 {}] regionserver.HRegionServer(1482): Serving as ae3311ea5ccb,37455,1731707679725, RpcServer on ae3311ea5ccb/172.17.0.3:37455, sessionid=0x100f86521fe0003 2024-11-15T21:54:41,264 DEBUG [RS:0;ae3311ea5ccb:40941 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-15T21:54:41,264 INFO [RS:1;ae3311ea5ccb:32871 {}] regionserver.HRegionServer(1482): Serving as ae3311ea5ccb,32871,1731707679650, RpcServer on ae3311ea5ccb/172.17.0.3:32871, sessionid=0x100f86521fe0002 2024-11-15T21:54:41,264 DEBUG [RS:2;ae3311ea5ccb:37455 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-15T21:54:41,264 DEBUG [RS:1;ae3311ea5ccb:32871 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-15T21:54:41,264 DEBUG [RS:2;ae3311ea5ccb:37455 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ae3311ea5ccb,37455,1731707679725 2024-11-15T21:54:41,264 DEBUG [RS:1;ae3311ea5ccb:32871 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ae3311ea5ccb,32871,1731707679650 2024-11-15T21:54:41,264 DEBUG [RS:2;ae3311ea5ccb:37455 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ae3311ea5ccb,37455,1731707679725' 2024-11-15T21:54:41,265 DEBUG [RS:1;ae3311ea5ccb:32871 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ae3311ea5ccb,32871,1731707679650' 2024-11-15T21:54:41,265 DEBUG [RS:2;ae3311ea5ccb:37455 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-15T21:54:41,265 DEBUG [RS:1;ae3311ea5ccb:32871 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-15T21:54:41,265 DEBUG [RS:0;ae3311ea5ccb:40941 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-15T21:54:41,265 DEBUG [RS:0;ae3311ea5ccb:40941 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-15T21:54:41,265 DEBUG [RS:0;ae3311ea5ccb:40941 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ae3311ea5ccb,40941,1731707679516 2024-11-15T21:54:41,265 DEBUG [RS:0;ae3311ea5ccb:40941 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ae3311ea5ccb,40941,1731707679516' 2024-11-15T21:54:41,265 DEBUG [RS:2;ae3311ea5ccb:37455 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-15T21:54:41,265 DEBUG [RS:0;ae3311ea5ccb:40941 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-15T21:54:41,266 DEBUG [RS:1;ae3311ea5ccb:32871 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-15T21:54:41,266 DEBUG [RS:0;ae3311ea5ccb:40941 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-15T21:54:41,266 DEBUG [RS:1;ae3311ea5ccb:32871 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-15T21:54:41,266 DEBUG [RS:2;ae3311ea5ccb:37455 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-15T21:54:41,267 DEBUG [RS:1;ae3311ea5ccb:32871 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-15T21:54:41,267 DEBUG [RS:2;ae3311ea5ccb:37455 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-15T21:54:41,267 DEBUG [RS:1;ae3311ea5ccb:32871 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ae3311ea5ccb,32871,1731707679650 2024-11-15T21:54:41,267 DEBUG [RS:2;ae3311ea5ccb:37455 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ae3311ea5ccb,37455,1731707679725 2024-11-15T21:54:41,267 DEBUG [RS:2;ae3311ea5ccb:37455 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ae3311ea5ccb,37455,1731707679725' 2024-11-15T21:54:41,267 DEBUG [RS:2;ae3311ea5ccb:37455 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-15T21:54:41,267 DEBUG [RS:1;ae3311ea5ccb:32871 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ae3311ea5ccb,32871,1731707679650' 2024-11-15T21:54:41,267 DEBUG [RS:0;ae3311ea5ccb:40941 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-15T21:54:41,267 DEBUG [RS:1;ae3311ea5ccb:32871 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-15T21:54:41,267 INFO [RS:0;ae3311ea5ccb:40941 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-15T21:54:41,267 INFO [RS:0;ae3311ea5ccb:40941 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-15T21:54:41,268 DEBUG [RS:2;ae3311ea5ccb:37455 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-15T21:54:41,268 DEBUG [RS:1;ae3311ea5ccb:32871 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-15T21:54:41,268 DEBUG [RS:2;ae3311ea5ccb:37455 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-15T21:54:41,268 INFO [RS:2;ae3311ea5ccb:37455 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-15T21:54:41,268 INFO [RS:2;ae3311ea5ccb:37455 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-15T21:54:41,269 DEBUG [RS:1;ae3311ea5ccb:32871 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-15T21:54:41,269 INFO [RS:1;ae3311ea5ccb:32871 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-15T21:54:41,269 INFO [RS:1;ae3311ea5ccb:32871 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-15T21:54:41,310 WARN [ae3311ea5ccb:36531 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-15T21:54:41,374 INFO [RS:2;ae3311ea5ccb:37455 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T21:54:41,374 INFO [RS:1;ae3311ea5ccb:32871 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T21:54:41,374 INFO [RS:0;ae3311ea5ccb:40941 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T21:54:41,378 INFO [RS:0;ae3311ea5ccb:40941 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ae3311ea5ccb%2C40941%2C1731707679516, suffix=, logDir=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/WALs/ae3311ea5ccb,40941,1731707679516, archiveDir=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/oldWALs, maxLogs=32 2024-11-15T21:54:41,378 INFO [RS:2;ae3311ea5ccb:37455 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ae3311ea5ccb%2C37455%2C1731707679725, suffix=, logDir=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/WALs/ae3311ea5ccb,37455,1731707679725, archiveDir=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/oldWALs, maxLogs=32 2024-11-15T21:54:41,379 INFO [RS:1;ae3311ea5ccb:32871 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ae3311ea5ccb%2C32871%2C1731707679650, suffix=, logDir=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/WALs/ae3311ea5ccb,32871,1731707679650, archiveDir=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/oldWALs, maxLogs=32 2024-11-15T21:54:41,400 DEBUG [RS:2;ae3311ea5ccb:37455 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/WALs/ae3311ea5ccb,37455,1731707679725/ae3311ea5ccb%2C37455%2C1731707679725.1731707681383, exclude list is [], retry=0 2024-11-15T21:54:41,403 DEBUG [RS:0;ae3311ea5ccb:40941 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/WALs/ae3311ea5ccb,40941,1731707679516/ae3311ea5ccb%2C40941%2C1731707679516.1731707681383, exclude list is [], retry=0 2024-11-15T21:54:41,406 DEBUG [RS:1;ae3311ea5ccb:32871 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/WALs/ae3311ea5ccb,32871,1731707679650/ae3311ea5ccb%2C32871%2C1731707679650.1731707681383, exclude list is [], retry=0 2024-11-15T21:54:41,406 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41653,DS-f55f5b18-f6a1-4c42-97ca-ec4b858b4b31,DISK] 2024-11-15T21:54:41,406 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40887,DS-cfe332f0-c324-40d1-87f6-7313ac5c3921,DISK] 2024-11-15T21:54:41,409 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36461,DS-34745d98-d41f-404b-baa7-624040cb25c7,DISK] 2024-11-15T21:54:41,409 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40887,DS-cfe332f0-c324-40d1-87f6-7313ac5c3921,DISK] 2024-11-15T21:54:41,410 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36461,DS-34745d98-d41f-404b-baa7-624040cb25c7,DISK] 2024-11-15T21:54:41,410 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41653,DS-f55f5b18-f6a1-4c42-97ca-ec4b858b4b31,DISK] 2024-11-15T21:54:41,415 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40887,DS-cfe332f0-c324-40d1-87f6-7313ac5c3921,DISK] 2024-11-15T21:54:41,415 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36461,DS-34745d98-d41f-404b-baa7-624040cb25c7,DISK] 2024-11-15T21:54:41,415 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41653,DS-f55f5b18-f6a1-4c42-97ca-ec4b858b4b31,DISK] 2024-11-15T21:54:41,425 INFO [RS:2;ae3311ea5ccb:37455 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/WALs/ae3311ea5ccb,37455,1731707679725/ae3311ea5ccb%2C37455%2C1731707679725.1731707681383 2024-11-15T21:54:41,426 INFO [RS:1;ae3311ea5ccb:32871 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/WALs/ae3311ea5ccb,32871,1731707679650/ae3311ea5ccb%2C32871%2C1731707679650.1731707681383 2024-11-15T21:54:41,426 DEBUG [RS:2;ae3311ea5ccb:37455 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44379:44379),(127.0.0.1/127.0.0.1:46569:46569),(127.0.0.1/127.0.0.1:41837:41837)] 2024-11-15T21:54:41,426 DEBUG [RS:1;ae3311ea5ccb:32871 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44379:44379),(127.0.0.1/127.0.0.1:41837:41837),(127.0.0.1/127.0.0.1:46569:46569)] 2024-11-15T21:54:41,426 INFO [RS:0;ae3311ea5ccb:40941 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/WALs/ae3311ea5ccb,40941,1731707679516/ae3311ea5ccb%2C40941%2C1731707679516.1731707681383 2024-11-15T21:54:41,427 DEBUG [RS:0;ae3311ea5ccb:40941 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44379:44379),(127.0.0.1/127.0.0.1:46569:46569),(127.0.0.1/127.0.0.1:41837:41837)] 2024-11-15T21:54:41,563 DEBUG [ae3311ea5ccb:36531 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-15T21:54:41,571 DEBUG [ae3311ea5ccb:36531 {}] balancer.BalancerClusterState(204): Hosts are {ae3311ea5ccb=0} racks are {/default-rack=0} 2024-11-15T21:54:41,578 DEBUG [ae3311ea5ccb:36531 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-15T21:54:41,578 DEBUG [ae3311ea5ccb:36531 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-15T21:54:41,578 DEBUG [ae3311ea5ccb:36531 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-15T21:54:41,578 DEBUG [ae3311ea5ccb:36531 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-15T21:54:41,578 DEBUG [ae3311ea5ccb:36531 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-15T21:54:41,578 DEBUG [ae3311ea5ccb:36531 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-15T21:54:41,578 INFO [ae3311ea5ccb:36531 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-15T21:54:41,578 INFO [ae3311ea5ccb:36531 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-15T21:54:41,578 INFO [ae3311ea5ccb:36531 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-15T21:54:41,578 DEBUG [ae3311ea5ccb:36531 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-15T21:54:41,586 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ae3311ea5ccb,32871,1731707679650 2024-11-15T21:54:41,593 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ae3311ea5ccb,32871,1731707679650, state=OPENING 2024-11-15T21:54:41,599 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-15T21:54:41,601 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32871-0x100f86521fe0002, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:54:41,601 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37455-0x100f86521fe0003, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:54:41,601 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36531-0x100f86521fe0000, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:54:41,601 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40941-0x100f86521fe0001, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:54:41,602 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T21:54:41,602 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T21:54:41,602 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T21:54:41,603 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T21:54:41,604 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T21:54:41,607 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=ae3311ea5ccb,32871,1731707679650}] 2024-11-15T21:54:41,785 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-15T21:54:41,788 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42751, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-15T21:54:41,800 INFO [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-15T21:54:41,801 INFO [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T21:54:41,801 INFO [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-15T21:54:41,804 INFO [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ae3311ea5ccb%2C32871%2C1731707679650.meta, suffix=.meta, logDir=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/WALs/ae3311ea5ccb,32871,1731707679650, archiveDir=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/oldWALs, maxLogs=32 2024-11-15T21:54:41,821 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/WALs/ae3311ea5ccb,32871,1731707679650/ae3311ea5ccb%2C32871%2C1731707679650.meta.1731707681806.meta, exclude list is [], retry=0 2024-11-15T21:54:41,826 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41653,DS-f55f5b18-f6a1-4c42-97ca-ec4b858b4b31,DISK] 2024-11-15T21:54:41,826 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36461,DS-34745d98-d41f-404b-baa7-624040cb25c7,DISK] 2024-11-15T21:54:41,826 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40887,DS-cfe332f0-c324-40d1-87f6-7313ac5c3921,DISK] 2024-11-15T21:54:41,829 INFO [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/WALs/ae3311ea5ccb,32871,1731707679650/ae3311ea5ccb%2C32871%2C1731707679650.meta.1731707681806.meta 2024-11-15T21:54:41,832 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46569:46569),(127.0.0.1/127.0.0.1:41837:41837),(127.0.0.1/127.0.0.1:44379:44379)] 2024-11-15T21:54:41,832 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-15T21:54:41,835 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-15T21:54:41,838 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-15T21:54:41,845 INFO [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-15T21:54:41,851 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-15T21:54:41,851 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:41,852 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-15T21:54:41,852 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-15T21:54:41,856 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-15T21:54:41,858 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-15T21:54:41,858 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:41,859 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T21:54:41,859 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-15T21:54:41,860 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-15T21:54:41,861 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:41,861 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T21:54:41,862 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-15T21:54:41,863 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-15T21:54:41,863 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:41,864 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T21:54:41,865 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-15T21:54:41,866 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-15T21:54:41,866 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:41,867 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T21:54:41,867 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-15T21:54:41,869 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/hbase/meta/1588230740 2024-11-15T21:54:41,872 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/hbase/meta/1588230740 2024-11-15T21:54:41,875 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-15T21:54:41,875 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-15T21:54:41,876 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-15T21:54:41,879 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-15T21:54:41,882 INFO [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64140781, jitterRate=-0.04422788321971893}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-15T21:54:41,882 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-15T21:54:41,884 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731707681852Writing region info on filesystem at 1731707681853 (+1 ms)Initializing all the Stores at 1731707681855 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731707681855Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731707681856 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707681856Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731707681856Cleaning up temporary data from old regions at 1731707681875 (+19 ms)Running coprocessor post-open hooks at 1731707681882 (+7 ms)Region opened successfully at 1731707681884 (+2 ms) 2024-11-15T21:54:41,895 INFO [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731707681773 2024-11-15T21:54:41,909 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-15T21:54:41,909 INFO [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-15T21:54:41,911 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=ae3311ea5ccb,32871,1731707679650 2024-11-15T21:54:41,914 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ae3311ea5ccb,32871,1731707679650, state=OPEN 2024-11-15T21:54:41,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37455-0x100f86521fe0003, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T21:54:41,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32871-0x100f86521fe0002, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T21:54:41,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36531-0x100f86521fe0000, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T21:54:41,917 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T21:54:41,917 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T21:54:41,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40941-0x100f86521fe0001, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T21:54:41,917 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T21:54:41,917 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T21:54:41,917 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=ae3311ea5ccb,32871,1731707679650 2024-11-15T21:54:41,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-15T21:54:41,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ae3311ea5ccb,32871,1731707679650 in 310 msec 2024-11-15T21:54:41,930 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-15T21:54:41,930 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 796 msec 2024-11-15T21:54:41,932 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T21:54:41,932 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-15T21:54:41,958 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T21:54:41,961 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1] 2024-11-15T21:54:41,990 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T21:54:41,993 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33601, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T21:54:42,019 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2300 sec 2024-11-15T21:54:42,019 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731707682019, completionTime=-1 2024-11-15T21:54:42,022 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-15T21:54:42,022 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-15T21:54:42,056 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-15T21:54:42,056 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731707742056 2024-11-15T21:54:42,056 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731707802056 2024-11-15T21:54:42,056 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 33 msec 2024-11-15T21:54:42,058 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-15T21:54:42,066 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ae3311ea5ccb,36531,1731707678636-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T21:54:42,066 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ae3311ea5ccb,36531,1731707678636-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T21:54:42,066 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ae3311ea5ccb,36531,1731707678636-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T21:54:42,068 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-ae3311ea5ccb:36531, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T21:54:42,069 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-15T21:54:42,070 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-15T21:54:42,077 DEBUG [master/ae3311ea5ccb:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-15T21:54:42,104 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.270sec 2024-11-15T21:54:42,106 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-15T21:54:42,108 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-15T21:54:42,109 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-15T21:54:42,109 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-15T21:54:42,109 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-15T21:54:42,110 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ae3311ea5ccb,36531,1731707678636-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T21:54:42,111 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ae3311ea5ccb,36531,1731707678636-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-15T21:54:42,116 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-15T21:54:42,118 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-15T21:54:42,118 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ae3311ea5ccb,36531,1731707678636-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T21:54:42,189 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f113c0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T21:54:42,194 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-15T21:54:42,195 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-15T21:54:42,200 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,36531,-1 for getting cluster id 2024-11-15T21:54:42,203 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T21:54:42,213 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4680bf58-6f3b-4f68-8171-8cc0087dc31c' 2024-11-15T21:54:42,216 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T21:54:42,216 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4680bf58-6f3b-4f68-8171-8cc0087dc31c" 2024-11-15T21:54:42,219 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@713d8da4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T21:54:42,219 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,36531,-1] 2024-11-15T21:54:42,222 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T21:54:42,225 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T21:54:42,226 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33880, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T21:54:42,260 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4225f0d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T21:54:42,261 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T21:54:42,271 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1] 2024-11-15T21:54:42,272 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T21:54:42,275 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48074, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T21:54:42,307 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=ae3311ea5ccb,36531,1731707678636 2024-11-15T21:54:42,309 WARN [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false 2024-11-15T21:54:42,315 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T21:54:42,320 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is ae3311ea5ccb,36531,1731707678636 2024-11-15T21:54:42,324 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1e2cacbb 2024-11-15T21:54:42,325 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T21:54:42,330 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33888, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T21:54:42,340 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36531 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T21:54:42,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36531 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-15T21:54:42,356 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T21:54:42,360 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:42,362 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36531 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true" procId is: 4 2024-11-15T21:54:42,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T21:54:42,370 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T21:54:42,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741837_1013 (size=1020) 2024-11-15T21:54:42,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741837_1013 (size=1020) 2024-11-15T21:54:42,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741837_1013 (size=1020) 2024-11-15T21:54:42,393 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 50df10b3ae47c5dc7c4521c4c58c192f, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731707682333.50df10b3ae47c5dc7c4521c4c58c192f.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad 2024-11-15T21:54:42,393 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => dd723478fcfc0e25b606965c9135244b, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731707682333.dd723478fcfc0e25b606965c9135244b.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad 2024-11-15T21:54:42,393 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 229a036fd38ae13e7359b15bc2f2e587, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731707682333.229a036fd38ae13e7359b15bc2f2e587.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad 2024-11-15T21:54:42,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741839_1015 (size=110) 2024-11-15T21:54:42,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741839_1015 (size=110) 2024-11-15T21:54:42,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741839_1015 (size=110) 2024-11-15T21:54:42,440 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731707682333.dd723478fcfc0e25b606965c9135244b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:42,440 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing dd723478fcfc0e25b606965c9135244b, disabling compactions & flushes 2024-11-15T21:54:42,440 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731707682333.dd723478fcfc0e25b606965c9135244b. 2024-11-15T21:54:42,440 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731707682333.dd723478fcfc0e25b606965c9135244b. 2024-11-15T21:54:42,440 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731707682333.dd723478fcfc0e25b606965c9135244b. after waiting 0 ms 2024-11-15T21:54:42,440 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731707682333.dd723478fcfc0e25b606965c9135244b. 2024-11-15T21:54:42,440 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731707682333.dd723478fcfc0e25b606965c9135244b. 2024-11-15T21:54:42,440 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for dd723478fcfc0e25b606965c9135244b: Waiting for close lock at 1731707682440Disabling compacts and flushes for region at 1731707682440Disabling writes for close at 1731707682440Writing region close event to WAL at 1731707682440Closed at 1731707682440 2024-11-15T21:54:42,441 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 4256fff2fa33299a6264dd460a4003fb, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731707682333.4256fff2fa33299a6264dd460a4003fb.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad 2024-11-15T21:54:42,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741838_1014 (size=110) 2024-11-15T21:54:42,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741838_1014 (size=110) 2024-11-15T21:54:42,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741838_1014 (size=110) 2024-11-15T21:54:42,447 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731707682333.50df10b3ae47c5dc7c4521c4c58c192f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:42,448 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing 50df10b3ae47c5dc7c4521c4c58c192f, disabling compactions & flushes 2024-11-15T21:54:42,448 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731707682333.50df10b3ae47c5dc7c4521c4c58c192f. 2024-11-15T21:54:42,448 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731707682333.50df10b3ae47c5dc7c4521c4c58c192f. 2024-11-15T21:54:42,448 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731707682333.50df10b3ae47c5dc7c4521c4c58c192f. after waiting 0 ms 2024-11-15T21:54:42,448 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731707682333.50df10b3ae47c5dc7c4521c4c58c192f. 2024-11-15T21:54:42,448 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731707682333.50df10b3ae47c5dc7c4521c4c58c192f. 2024-11-15T21:54:42,448 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for 50df10b3ae47c5dc7c4521c4c58c192f: Waiting for close lock at 1731707682448Disabling compacts and flushes for region at 1731707682448Disabling writes for close at 1731707682448Writing region close event to WAL at 1731707682448Closed at 1731707682448 2024-11-15T21:54:42,449 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 2c83200b2c422973a68afce1173aae30, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731707682333.2c83200b2c422973a68afce1173aae30.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad 2024-11-15T21:54:42,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741840_1016 (size=107) 2024-11-15T21:54:42,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741840_1016 (size=107) 2024-11-15T21:54:42,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741840_1016 (size=107) 2024-11-15T21:54:42,458 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731707682333.229a036fd38ae13e7359b15bc2f2e587.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:42,458 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 229a036fd38ae13e7359b15bc2f2e587, disabling compactions & flushes 2024-11-15T21:54:42,458 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731707682333.229a036fd38ae13e7359b15bc2f2e587. 2024-11-15T21:54:42,458 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731707682333.229a036fd38ae13e7359b15bc2f2e587. 2024-11-15T21:54:42,458 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731707682333.229a036fd38ae13e7359b15bc2f2e587. after waiting 0 ms 2024-11-15T21:54:42,458 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731707682333.229a036fd38ae13e7359b15bc2f2e587. 2024-11-15T21:54:42,458 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731707682333.229a036fd38ae13e7359b15bc2f2e587. 2024-11-15T21:54:42,458 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 229a036fd38ae13e7359b15bc2f2e587: Waiting for close lock at 1731707682458Disabling compacts and flushes for region at 1731707682458Disabling writes for close at 1731707682458Writing region close event to WAL at 1731707682458Closed at 1731707682458 2024-11-15T21:54:42,459 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3f13d1331edcf64c426c3bf9433330e4, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731707682333.3f13d1331edcf64c426c3bf9433330e4.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad 2024-11-15T21:54:42,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741841_1017 (size=110) 2024-11-15T21:54:42,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741841_1017 (size=110) 2024-11-15T21:54:42,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741841_1017 (size=110) 2024-11-15T21:54:42,467 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731707682333.4256fff2fa33299a6264dd460a4003fb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:42,467 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing 4256fff2fa33299a6264dd460a4003fb, disabling compactions & flushes 2024-11-15T21:54:42,467 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731707682333.4256fff2fa33299a6264dd460a4003fb. 2024-11-15T21:54:42,467 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731707682333.4256fff2fa33299a6264dd460a4003fb. 2024-11-15T21:54:42,467 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731707682333.4256fff2fa33299a6264dd460a4003fb. after waiting 0 ms 2024-11-15T21:54:42,467 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731707682333.4256fff2fa33299a6264dd460a4003fb. 2024-11-15T21:54:42,467 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731707682333.4256fff2fa33299a6264dd460a4003fb. 2024-11-15T21:54:42,468 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for 4256fff2fa33299a6264dd460a4003fb: Waiting for close lock at 1731707682467Disabling compacts and flushes for region at 1731707682467Disabling writes for close at 1731707682467Writing region close event to WAL at 1731707682467Closed at 1731707682467 2024-11-15T21:54:42,468 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => f9ef867a298d175eca9d85304f2faa69, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731707682333.f9ef867a298d175eca9d85304f2faa69.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad 2024-11-15T21:54:42,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741842_1018 (size=110) 2024-11-15T21:54:42,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741842_1018 (size=110) 2024-11-15T21:54:42,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741842_1018 (size=110) 2024-11-15T21:54:42,473 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731707682333.2c83200b2c422973a68afce1173aae30.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:42,473 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing 2c83200b2c422973a68afce1173aae30, disabling compactions & flushes 2024-11-15T21:54:42,474 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731707682333.2c83200b2c422973a68afce1173aae30. 2024-11-15T21:54:42,474 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731707682333.2c83200b2c422973a68afce1173aae30. 2024-11-15T21:54:42,474 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731707682333.2c83200b2c422973a68afce1173aae30. after waiting 0 ms 2024-11-15T21:54:42,474 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731707682333.2c83200b2c422973a68afce1173aae30. 2024-11-15T21:54:42,474 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731707682333.2c83200b2c422973a68afce1173aae30. 2024-11-15T21:54:42,474 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for 2c83200b2c422973a68afce1173aae30: Waiting for close lock at 1731707682473Disabling compacts and flushes for region at 1731707682473Disabling writes for close at 1731707682474 (+1 ms)Writing region close event to WAL at 1731707682474Closed at 1731707682474 2024-11-15T21:54:42,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741843_1019 (size=110) 2024-11-15T21:54:42,475 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 0890aeec1f0ca0649e6483a2d56f36b4, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731707682333.0890aeec1f0ca0649e6483a2d56f36b4.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad 2024-11-15T21:54:42,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741843_1019 (size=110) 2024-11-15T21:54:42,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T21:54:42,480 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731707682333.3f13d1331edcf64c426c3bf9433330e4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:42,480 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 3f13d1331edcf64c426c3bf9433330e4, disabling compactions & flushes 2024-11-15T21:54:42,481 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731707682333.3f13d1331edcf64c426c3bf9433330e4. 2024-11-15T21:54:42,481 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731707682333.3f13d1331edcf64c426c3bf9433330e4. 2024-11-15T21:54:42,481 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731707682333.3f13d1331edcf64c426c3bf9433330e4. after waiting 0 ms 2024-11-15T21:54:42,481 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731707682333.3f13d1331edcf64c426c3bf9433330e4. 2024-11-15T21:54:42,481 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731707682333.3f13d1331edcf64c426c3bf9433330e4. 2024-11-15T21:54:42,481 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3f13d1331edcf64c426c3bf9433330e4: Waiting for close lock at 1731707682480Disabling compacts and flushes for region at 1731707682480Disabling writes for close at 1731707682481 (+1 ms)Writing region close event to WAL at 1731707682481Closed at 1731707682481 2024-11-15T21:54:42,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741843_1019 (size=110) 2024-11-15T21:54:42,482 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d28ee8a2ac06d2199c72ac7f884ba470, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731707682333.d28ee8a2ac06d2199c72ac7f884ba470.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad 2024-11-15T21:54:42,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741844_1020 (size=110) 2024-11-15T21:54:42,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741844_1020 (size=110) 2024-11-15T21:54:42,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741844_1020 (size=110) 2024-11-15T21:54:42,502 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731707682333.f9ef867a298d175eca9d85304f2faa69.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:42,502 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing f9ef867a298d175eca9d85304f2faa69, disabling compactions & flushes 2024-11-15T21:54:42,502 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731707682333.f9ef867a298d175eca9d85304f2faa69. 2024-11-15T21:54:42,502 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731707682333.f9ef867a298d175eca9d85304f2faa69. 2024-11-15T21:54:42,502 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731707682333.f9ef867a298d175eca9d85304f2faa69. after waiting 0 ms 2024-11-15T21:54:42,502 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731707682333.f9ef867a298d175eca9d85304f2faa69. 2024-11-15T21:54:42,502 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731707682333.f9ef867a298d175eca9d85304f2faa69. 2024-11-15T21:54:42,503 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for f9ef867a298d175eca9d85304f2faa69: Waiting for close lock at 1731707682502Disabling compacts and flushes for region at 1731707682502Disabling writes for close at 1731707682502Writing region close event to WAL at 1731707682502Closed at 1731707682502 2024-11-15T21:54:42,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741845_1021 (size=110) 2024-11-15T21:54:42,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741845_1021 (size=110) 2024-11-15T21:54:42,504 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => aec2e46cb64898ba2a66825d3732f6dc, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731707682333.aec2e46cb64898ba2a66825d3732f6dc.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad 2024-11-15T21:54:42,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741845_1021 (size=110) 2024-11-15T21:54:42,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741846_1022 (size=110) 2024-11-15T21:54:42,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741846_1022 (size=110) 2024-11-15T21:54:42,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741846_1022 (size=110) 2024-11-15T21:54:42,507 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731707682333.0890aeec1f0ca0649e6483a2d56f36b4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:42,507 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing 0890aeec1f0ca0649e6483a2d56f36b4, disabling compactions & flushes 2024-11-15T21:54:42,507 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731707682333.0890aeec1f0ca0649e6483a2d56f36b4. 2024-11-15T21:54:42,507 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731707682333.0890aeec1f0ca0649e6483a2d56f36b4. 2024-11-15T21:54:42,507 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731707682333.0890aeec1f0ca0649e6483a2d56f36b4. after waiting 0 ms 2024-11-15T21:54:42,507 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731707682333.0890aeec1f0ca0649e6483a2d56f36b4. 2024-11-15T21:54:42,507 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731707682333.0890aeec1f0ca0649e6483a2d56f36b4. 2024-11-15T21:54:42,508 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for 0890aeec1f0ca0649e6483a2d56f36b4: Waiting for close lock at 1731707682507Disabling compacts and flushes for region at 1731707682507Disabling writes for close at 1731707682507Writing region close event to WAL at 1731707682507Closed at 1731707682507 2024-11-15T21:54:42,508 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => cbb6b5b8341f7ff2ada4ac1931a1a588, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731707682333.cbb6b5b8341f7ff2ada4ac1931a1a588.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad 2024-11-15T21:54:42,509 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731707682333.d28ee8a2ac06d2199c72ac7f884ba470.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:42,509 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing d28ee8a2ac06d2199c72ac7f884ba470, disabling compactions & flushes 2024-11-15T21:54:42,509 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731707682333.d28ee8a2ac06d2199c72ac7f884ba470. 2024-11-15T21:54:42,509 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731707682333.d28ee8a2ac06d2199c72ac7f884ba470. 2024-11-15T21:54:42,509 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731707682333.d28ee8a2ac06d2199c72ac7f884ba470. after waiting 0 ms 2024-11-15T21:54:42,509 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731707682333.d28ee8a2ac06d2199c72ac7f884ba470. 2024-11-15T21:54:42,509 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731707682333.d28ee8a2ac06d2199c72ac7f884ba470. 2024-11-15T21:54:42,510 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for d28ee8a2ac06d2199c72ac7f884ba470: Waiting for close lock at 1731707682509Disabling compacts and flushes for region at 1731707682509Disabling writes for close at 1731707682509Writing region close event to WAL at 1731707682509Closed at 1731707682509 2024-11-15T21:54:42,510 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a3f7be24d811f4d6177c28e08f25af2d, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731707682333.a3f7be24d811f4d6177c28e08f25af2d.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad 2024-11-15T21:54:42,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741847_1023 (size=110) 2024-11-15T21:54:42,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741847_1023 (size=110) 2024-11-15T21:54:42,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741847_1023 (size=110) 2024-11-15T21:54:42,551 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731707682333.aec2e46cb64898ba2a66825d3732f6dc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:42,551 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing aec2e46cb64898ba2a66825d3732f6dc, disabling compactions & flushes 2024-11-15T21:54:42,551 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731707682333.aec2e46cb64898ba2a66825d3732f6dc. 2024-11-15T21:54:42,551 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731707682333.aec2e46cb64898ba2a66825d3732f6dc. 2024-11-15T21:54:42,551 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731707682333.aec2e46cb64898ba2a66825d3732f6dc. after waiting 0 ms 2024-11-15T21:54:42,551 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731707682333.aec2e46cb64898ba2a66825d3732f6dc. 2024-11-15T21:54:42,551 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731707682333.aec2e46cb64898ba2a66825d3732f6dc. 2024-11-15T21:54:42,551 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for aec2e46cb64898ba2a66825d3732f6dc: Waiting for close lock at 1731707682551Disabling compacts and flushes for region at 1731707682551Disabling writes for close at 1731707682551Writing region close event to WAL at 1731707682551Closed at 1731707682551 2024-11-15T21:54:42,552 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 6e3b749fcbfdadfe59f1d1cf94f5406a, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731707682333.6e3b749fcbfdadfe59f1d1cf94f5406a.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad 2024-11-15T21:54:42,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741848_1024 (size=110) 2024-11-15T21:54:42,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741848_1024 (size=110) 2024-11-15T21:54:42,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741848_1024 (size=110) 2024-11-15T21:54:42,561 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731707682333.cbb6b5b8341f7ff2ada4ac1931a1a588.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:42,561 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing cbb6b5b8341f7ff2ada4ac1931a1a588, disabling compactions & flushes 2024-11-15T21:54:42,561 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731707682333.cbb6b5b8341f7ff2ada4ac1931a1a588. 2024-11-15T21:54:42,562 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731707682333.cbb6b5b8341f7ff2ada4ac1931a1a588. 2024-11-15T21:54:42,562 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731707682333.cbb6b5b8341f7ff2ada4ac1931a1a588. after waiting 0 ms 2024-11-15T21:54:42,562 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731707682333.cbb6b5b8341f7ff2ada4ac1931a1a588. 2024-11-15T21:54:42,562 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731707682333.cbb6b5b8341f7ff2ada4ac1931a1a588. 2024-11-15T21:54:42,562 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for cbb6b5b8341f7ff2ada4ac1931a1a588: Waiting for close lock at 1731707682561Disabling compacts and flushes for region at 1731707682561Disabling writes for close at 1731707682562 (+1 ms)Writing region close event to WAL at 1731707682562Closed at 1731707682562 2024-11-15T21:54:42,563 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 51af260ba7a9b40a7f788a5c93a416b6, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731707682333.51af260ba7a9b40a7f788a5c93a416b6.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad 2024-11-15T21:54:42,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741849_1025 (size=110) 2024-11-15T21:54:42,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741849_1025 (size=110) 2024-11-15T21:54:42,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741849_1025 (size=110) 2024-11-15T21:54:42,575 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731707682333.a3f7be24d811f4d6177c28e08f25af2d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:42,575 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing a3f7be24d811f4d6177c28e08f25af2d, disabling compactions & flushes 2024-11-15T21:54:42,575 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731707682333.a3f7be24d811f4d6177c28e08f25af2d. 2024-11-15T21:54:42,575 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731707682333.a3f7be24d811f4d6177c28e08f25af2d. 2024-11-15T21:54:42,575 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731707682333.a3f7be24d811f4d6177c28e08f25af2d. after waiting 0 ms 2024-11-15T21:54:42,575 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731707682333.a3f7be24d811f4d6177c28e08f25af2d. 2024-11-15T21:54:42,575 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731707682333.a3f7be24d811f4d6177c28e08f25af2d. 2024-11-15T21:54:42,575 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for a3f7be24d811f4d6177c28e08f25af2d: Waiting for close lock at 1731707682575Disabling compacts and flushes for region at 1731707682575Disabling writes for close at 1731707682575Writing region close event to WAL at 1731707682575Closed at 1731707682575 2024-11-15T21:54:42,576 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0ab2d104e772f41ee9c56851adad43f0, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731707682333.0ab2d104e772f41ee9c56851adad43f0.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad 2024-11-15T21:54:42,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741850_1026 (size=110) 2024-11-15T21:54:42,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741850_1026 (size=110) 2024-11-15T21:54:42,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741850_1026 (size=110) 2024-11-15T21:54:42,592 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731707682333.6e3b749fcbfdadfe59f1d1cf94f5406a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:42,592 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing 6e3b749fcbfdadfe59f1d1cf94f5406a, disabling compactions & flushes 2024-11-15T21:54:42,592 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731707682333.6e3b749fcbfdadfe59f1d1cf94f5406a. 2024-11-15T21:54:42,592 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731707682333.6e3b749fcbfdadfe59f1d1cf94f5406a. 2024-11-15T21:54:42,592 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731707682333.6e3b749fcbfdadfe59f1d1cf94f5406a. after waiting 0 ms 2024-11-15T21:54:42,593 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731707682333.6e3b749fcbfdadfe59f1d1cf94f5406a. 2024-11-15T21:54:42,593 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731707682333.6e3b749fcbfdadfe59f1d1cf94f5406a. 2024-11-15T21:54:42,593 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for 6e3b749fcbfdadfe59f1d1cf94f5406a: Waiting for close lock at 1731707682592Disabling compacts and flushes for region at 1731707682592Disabling writes for close at 1731707682592Writing region close event to WAL at 1731707682593 (+1 ms)Closed at 1731707682593 2024-11-15T21:54:42,594 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 3cf264afea186a5cc96edd3e68a10d45, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731707682333.3cf264afea186a5cc96edd3e68a10d45.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad 2024-11-15T21:54:42,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741851_1027 (size=110) 2024-11-15T21:54:42,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741851_1027 (size=110) 2024-11-15T21:54:42,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741851_1027 (size=110) 2024-11-15T21:54:42,600 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731707682333.51af260ba7a9b40a7f788a5c93a416b6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:42,600 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing 51af260ba7a9b40a7f788a5c93a416b6, disabling compactions & flushes 2024-11-15T21:54:42,600 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731707682333.51af260ba7a9b40a7f788a5c93a416b6. 2024-11-15T21:54:42,600 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731707682333.51af260ba7a9b40a7f788a5c93a416b6. 2024-11-15T21:54:42,600 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731707682333.51af260ba7a9b40a7f788a5c93a416b6. after waiting 0 ms 2024-11-15T21:54:42,600 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731707682333.51af260ba7a9b40a7f788a5c93a416b6. 2024-11-15T21:54:42,600 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731707682333.51af260ba7a9b40a7f788a5c93a416b6. 2024-11-15T21:54:42,600 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for 51af260ba7a9b40a7f788a5c93a416b6: Waiting for close lock at 1731707682600Disabling compacts and flushes for region at 1731707682600Disabling writes for close at 1731707682600Writing region close event to WAL at 1731707682600Closed at 1731707682600 2024-11-15T21:54:42,601 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => d100add17d87270d000a3c39b49f8227, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731707682333.d100add17d87270d000a3c39b49f8227.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad 2024-11-15T21:54:42,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741852_1028 (size=110) 2024-11-15T21:54:42,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741852_1028 (size=110) 2024-11-15T21:54:42,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741852_1028 (size=110) 2024-11-15T21:54:42,605 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731707682333.0ab2d104e772f41ee9c56851adad43f0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:42,605 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 0ab2d104e772f41ee9c56851adad43f0, disabling compactions & flushes 2024-11-15T21:54:42,605 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731707682333.0ab2d104e772f41ee9c56851adad43f0. 2024-11-15T21:54:42,605 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731707682333.0ab2d104e772f41ee9c56851adad43f0. 2024-11-15T21:54:42,606 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731707682333.0ab2d104e772f41ee9c56851adad43f0. after waiting 0 ms 2024-11-15T21:54:42,606 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731707682333.0ab2d104e772f41ee9c56851adad43f0. 2024-11-15T21:54:42,606 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731707682333.0ab2d104e772f41ee9c56851adad43f0. 2024-11-15T21:54:42,606 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0ab2d104e772f41ee9c56851adad43f0: Waiting for close lock at 1731707682605Disabling compacts and flushes for region at 1731707682605Disabling writes for close at 1731707682606 (+1 ms)Writing region close event to WAL at 1731707682606Closed at 1731707682606 2024-11-15T21:54:42,607 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d9989b0341ef96818e811f874dbcf91c, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731707682333.d9989b0341ef96818e811f874dbcf91c.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad 2024-11-15T21:54:42,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741853_1029 (size=110) 2024-11-15T21:54:42,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741853_1029 (size=110) 2024-11-15T21:54:42,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741853_1029 (size=110) 2024-11-15T21:54:42,640 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731707682333.3cf264afea186a5cc96edd3e68a10d45.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:42,640 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing 3cf264afea186a5cc96edd3e68a10d45, disabling compactions & flushes 2024-11-15T21:54:42,640 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731707682333.3cf264afea186a5cc96edd3e68a10d45. 2024-11-15T21:54:42,640 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731707682333.3cf264afea186a5cc96edd3e68a10d45. 2024-11-15T21:54:42,640 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731707682333.3cf264afea186a5cc96edd3e68a10d45. after waiting 0 ms 2024-11-15T21:54:42,640 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731707682333.3cf264afea186a5cc96edd3e68a10d45. 2024-11-15T21:54:42,640 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731707682333.3cf264afea186a5cc96edd3e68a10d45. 2024-11-15T21:54:42,640 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for 3cf264afea186a5cc96edd3e68a10d45: Waiting for close lock at 1731707682640Disabling compacts and flushes for region at 1731707682640Disabling writes for close at 1731707682640Writing region close event to WAL at 1731707682640Closed at 1731707682640 2024-11-15T21:54:42,641 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 386f4af1e738f142f3fbde37773d4d2c, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731707682333.386f4af1e738f142f3fbde37773d4d2c.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad 2024-11-15T21:54:42,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741854_1030 (size=110) 2024-11-15T21:54:42,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741854_1030 (size=110) 2024-11-15T21:54:42,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741854_1030 (size=110) 2024-11-15T21:54:42,651 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731707682333.d100add17d87270d000a3c39b49f8227.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:42,651 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing d100add17d87270d000a3c39b49f8227, disabling compactions & flushes 2024-11-15T21:54:42,651 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731707682333.d100add17d87270d000a3c39b49f8227. 2024-11-15T21:54:42,651 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731707682333.d100add17d87270d000a3c39b49f8227. 2024-11-15T21:54:42,651 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731707682333.d100add17d87270d000a3c39b49f8227. after waiting 0 ms 2024-11-15T21:54:42,651 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731707682333.d100add17d87270d000a3c39b49f8227. 2024-11-15T21:54:42,652 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731707682333.d100add17d87270d000a3c39b49f8227. 2024-11-15T21:54:42,652 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for d100add17d87270d000a3c39b49f8227: Waiting for close lock at 1731707682651Disabling compacts and flushes for region at 1731707682651Disabling writes for close at 1731707682651Writing region close event to WAL at 1731707682651Closed at 1731707682651 2024-11-15T21:54:42,652 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => d6063c53955b74f03cc36bd578a6bd41, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731707682333.d6063c53955b74f03cc36bd578a6bd41.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad 2024-11-15T21:54:42,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741855_1031 (size=110) 2024-11-15T21:54:42,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741855_1031 (size=110) 2024-11-15T21:54:42,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741855_1031 (size=110) 2024-11-15T21:54:42,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741856_1032 (size=110) 2024-11-15T21:54:42,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741856_1032 (size=110) 2024-11-15T21:54:42,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741856_1032 (size=110) 2024-11-15T21:54:42,666 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731707682333.386f4af1e738f142f3fbde37773d4d2c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:42,666 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing 386f4af1e738f142f3fbde37773d4d2c, disabling compactions & flushes 2024-11-15T21:54:42,666 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731707682333.386f4af1e738f142f3fbde37773d4d2c. 2024-11-15T21:54:42,666 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731707682333.386f4af1e738f142f3fbde37773d4d2c. 2024-11-15T21:54:42,666 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731707682333.386f4af1e738f142f3fbde37773d4d2c. after waiting 0 ms 2024-11-15T21:54:42,667 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731707682333.386f4af1e738f142f3fbde37773d4d2c. 2024-11-15T21:54:42,667 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731707682333.386f4af1e738f142f3fbde37773d4d2c. 2024-11-15T21:54:42,667 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for 386f4af1e738f142f3fbde37773d4d2c: Waiting for close lock at 1731707682666Disabling compacts and flushes for region at 1731707682666Disabling writes for close at 1731707682666Writing region close event to WAL at 1731707682667 (+1 ms)Closed at 1731707682667 2024-11-15T21:54:42,668 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => fac6af87aa0fc9826df0515f766c031e, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731707682333.fac6af87aa0fc9826df0515f766c031e.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad 2024-11-15T21:54:42,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741857_1033 (size=110) 2024-11-15T21:54:42,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741857_1033 (size=110) 2024-11-15T21:54:42,676 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731707682333.d6063c53955b74f03cc36bd578a6bd41.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:42,676 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing d6063c53955b74f03cc36bd578a6bd41, disabling compactions & flushes 2024-11-15T21:54:42,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741857_1033 (size=110) 2024-11-15T21:54:42,677 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731707682333.d6063c53955b74f03cc36bd578a6bd41. 2024-11-15T21:54:42,677 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731707682333.d6063c53955b74f03cc36bd578a6bd41. 2024-11-15T21:54:42,677 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731707682333.d6063c53955b74f03cc36bd578a6bd41. after waiting 0 ms 2024-11-15T21:54:42,677 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731707682333.d6063c53955b74f03cc36bd578a6bd41. 2024-11-15T21:54:42,677 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731707682333.d6063c53955b74f03cc36bd578a6bd41. 2024-11-15T21:54:42,677 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for d6063c53955b74f03cc36bd578a6bd41: Waiting for close lock at 1731707682676Disabling compacts and flushes for region at 1731707682676Disabling writes for close at 1731707682677 (+1 ms)Writing region close event to WAL at 1731707682677Closed at 1731707682677 2024-11-15T21:54:42,678 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 5ce283f02f0974e39c98759bb2bc9883, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731707682333.5ce283f02f0974e39c98759bb2bc9883.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad 2024-11-15T21:54:42,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T21:54:42,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741858_1034 (size=110) 2024-11-15T21:54:42,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741858_1034 (size=110) 2024-11-15T21:54:42,698 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731707682333.fac6af87aa0fc9826df0515f766c031e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:42,698 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing fac6af87aa0fc9826df0515f766c031e, disabling compactions & flushes 2024-11-15T21:54:42,698 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731707682333.fac6af87aa0fc9826df0515f766c031e. 2024-11-15T21:54:42,698 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731707682333.fac6af87aa0fc9826df0515f766c031e. 2024-11-15T21:54:42,699 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731707682333.fac6af87aa0fc9826df0515f766c031e. after waiting 0 ms 2024-11-15T21:54:42,699 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731707682333.fac6af87aa0fc9826df0515f766c031e. 2024-11-15T21:54:42,699 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731707682333.fac6af87aa0fc9826df0515f766c031e. 2024-11-15T21:54:42,699 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for fac6af87aa0fc9826df0515f766c031e: Waiting for close lock at 1731707682698Disabling compacts and flushes for region at 1731707682698Disabling writes for close at 1731707682699 (+1 ms)Writing region close event to WAL at 1731707682699Closed at 1731707682699 2024-11-15T21:54:42,700 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 6694325f9c862a5a1ee02eabf7861cc3, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731707682333.6694325f9c862a5a1ee02eabf7861cc3.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad 2024-11-15T21:54:42,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741858_1034 (size=110) 2024-11-15T21:54:42,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741859_1035 (size=110) 2024-11-15T21:54:42,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741859_1035 (size=110) 2024-11-15T21:54:42,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741859_1035 (size=110) 2024-11-15T21:54:42,710 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731707682333.5ce283f02f0974e39c98759bb2bc9883.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:42,710 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing 5ce283f02f0974e39c98759bb2bc9883, disabling compactions & flushes 2024-11-15T21:54:42,710 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731707682333.5ce283f02f0974e39c98759bb2bc9883. 2024-11-15T21:54:42,710 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731707682333.5ce283f02f0974e39c98759bb2bc9883. 2024-11-15T21:54:42,710 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731707682333.5ce283f02f0974e39c98759bb2bc9883. after waiting 0 ms 2024-11-15T21:54:42,710 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731707682333.5ce283f02f0974e39c98759bb2bc9883. 2024-11-15T21:54:42,710 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731707682333.5ce283f02f0974e39c98759bb2bc9883. 2024-11-15T21:54:42,711 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for 5ce283f02f0974e39c98759bb2bc9883: Waiting for close lock at 1731707682710Disabling compacts and flushes for region at 1731707682710Disabling writes for close at 1731707682710Writing region close event to WAL at 1731707682710Closed at 1731707682710 2024-11-15T21:54:42,711 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 46b5f83abe9585829b26884a1b26b43f, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731707682333.46b5f83abe9585829b26884a1b26b43f.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad 2024-11-15T21:54:42,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741860_1036 (size=110) 2024-11-15T21:54:42,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741860_1036 (size=110) 2024-11-15T21:54:42,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741861_1037 (size=110) 2024-11-15T21:54:42,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741861_1037 (size=110) 2024-11-15T21:54:42,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741860_1036 (size=110) 2024-11-15T21:54:42,728 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731707682333.6694325f9c862a5a1ee02eabf7861cc3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:42,729 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing 6694325f9c862a5a1ee02eabf7861cc3, disabling compactions & flushes 2024-11-15T21:54:42,729 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731707682333.6694325f9c862a5a1ee02eabf7861cc3. 2024-11-15T21:54:42,729 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731707682333.6694325f9c862a5a1ee02eabf7861cc3. 2024-11-15T21:54:42,729 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731707682333.6694325f9c862a5a1ee02eabf7861cc3. after waiting 0 ms 2024-11-15T21:54:42,729 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731707682333.6694325f9c862a5a1ee02eabf7861cc3. 2024-11-15T21:54:42,729 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731707682333.6694325f9c862a5a1ee02eabf7861cc3. 2024-11-15T21:54:42,729 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for 6694325f9c862a5a1ee02eabf7861cc3: Waiting for close lock at 1731707682729Disabling compacts and flushes for region at 1731707682729Disabling writes for close at 1731707682729Writing region close event to WAL at 1731707682729Closed at 1731707682729 2024-11-15T21:54:42,730 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1031049c0b4149d1189db33e38ab56cd, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731707682333.1031049c0b4149d1189db33e38ab56cd.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad 2024-11-15T21:54:42,730 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731707682333.46b5f83abe9585829b26884a1b26b43f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:42,730 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing 46b5f83abe9585829b26884a1b26b43f, disabling compactions & flushes 2024-11-15T21:54:42,730 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731707682333.46b5f83abe9585829b26884a1b26b43f. 2024-11-15T21:54:42,730 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731707682333.46b5f83abe9585829b26884a1b26b43f. 2024-11-15T21:54:42,730 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731707682333.46b5f83abe9585829b26884a1b26b43f. after waiting 0 ms 2024-11-15T21:54:42,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741861_1037 (size=110) 2024-11-15T21:54:42,730 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731707682333.46b5f83abe9585829b26884a1b26b43f. 2024-11-15T21:54:42,730 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731707682333.46b5f83abe9585829b26884a1b26b43f. 2024-11-15T21:54:42,730 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for 46b5f83abe9585829b26884a1b26b43f: Waiting for close lock at 1731707682730Disabling compacts and flushes for region at 1731707682730Disabling writes for close at 1731707682730Writing region close event to WAL at 1731707682730Closed at 1731707682730 2024-11-15T21:54:42,731 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => e76ae188febdca91d319999f619d9e11, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731707682333.e76ae188febdca91d319999f619d9e11.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad 2024-11-15T21:54:42,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741863_1039 (size=107) 2024-11-15T21:54:42,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741863_1039 (size=107) 2024-11-15T21:54:42,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741863_1039 (size=107) 2024-11-15T21:54:42,753 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731707682333.e76ae188febdca91d319999f619d9e11.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:42,753 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing e76ae188febdca91d319999f619d9e11, disabling compactions & flushes 2024-11-15T21:54:42,753 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731707682333.e76ae188febdca91d319999f619d9e11. 2024-11-15T21:54:42,753 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731707682333.e76ae188febdca91d319999f619d9e11. 2024-11-15T21:54:42,753 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731707682333.e76ae188febdca91d319999f619d9e11. after waiting 0 ms 2024-11-15T21:54:42,753 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731707682333.e76ae188febdca91d319999f619d9e11. 2024-11-15T21:54:42,753 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731707682333.e76ae188febdca91d319999f619d9e11. 2024-11-15T21:54:42,753 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for e76ae188febdca91d319999f619d9e11: Waiting for close lock at 1731707682753Disabling compacts and flushes for region at 1731707682753Disabling writes for close at 1731707682753Writing region close event to WAL at 1731707682753Closed at 1731707682753 2024-11-15T21:54:42,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741862_1038 (size=110) 2024-11-15T21:54:42,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741862_1038 (size=110) 2024-11-15T21:54:42,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741862_1038 (size=110) 2024-11-15T21:54:42,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T21:54:43,060 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731707682333.d9989b0341ef96818e811f874dbcf91c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:43,061 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing d9989b0341ef96818e811f874dbcf91c, disabling compactions & flushes 2024-11-15T21:54:43,061 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731707682333.d9989b0341ef96818e811f874dbcf91c. 2024-11-15T21:54:43,061 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731707682333.d9989b0341ef96818e811f874dbcf91c. 2024-11-15T21:54:43,061 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731707682333.d9989b0341ef96818e811f874dbcf91c. after waiting 0 ms 2024-11-15T21:54:43,061 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731707682333.d9989b0341ef96818e811f874dbcf91c. 2024-11-15T21:54:43,061 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731707682333.d9989b0341ef96818e811f874dbcf91c. 2024-11-15T21:54:43,061 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for d9989b0341ef96818e811f874dbcf91c: Waiting for close lock at 1731707683061Disabling compacts and flushes for region at 1731707683061Disabling writes for close at 1731707683061Writing region close event to WAL at 1731707683061Closed at 1731707683061 2024-11-15T21:54:43,163 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731707682333.1031049c0b4149d1189db33e38ab56cd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:43,163 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing 1031049c0b4149d1189db33e38ab56cd, disabling compactions & flushes 2024-11-15T21:54:43,163 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731707682333.1031049c0b4149d1189db33e38ab56cd. 2024-11-15T21:54:43,163 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731707682333.1031049c0b4149d1189db33e38ab56cd. 2024-11-15T21:54:43,163 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731707682333.1031049c0b4149d1189db33e38ab56cd. after waiting 0 ms 2024-11-15T21:54:43,163 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731707682333.1031049c0b4149d1189db33e38ab56cd. 2024-11-15T21:54:43,163 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731707682333.1031049c0b4149d1189db33e38ab56cd. 2024-11-15T21:54:43,163 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for 1031049c0b4149d1189db33e38ab56cd: Waiting for close lock at 1731707683163Disabling compacts and flushes for region at 1731707683163Disabling writes for close at 1731707683163Writing region close event to WAL at 1731707683163Closed at 1731707683163 2024-11-15T21:54:43,166 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T21:54:43,173 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731707682333.dd723478fcfc0e25b606965c9135244b.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731707683167"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731707683167"}]},"ts":"1731707683167"} 2024-11-15T21:54:43,174 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731707682333.50df10b3ae47c5dc7c4521c4c58c192f.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731707683167"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731707683167"}]},"ts":"1731707683167"} 2024-11-15T21:54:43,174 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731707682333.229a036fd38ae13e7359b15bc2f2e587.","families":{"info":[{"qualifier":"regioninfo","vlen":106,"tag":[],"timestamp":"1731707683167"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731707683167"}]},"ts":"1731707683167"} 2024-11-15T21:54:43,174 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731707682333.4256fff2fa33299a6264dd460a4003fb.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731707683167"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731707683167"}]},"ts":"1731707683167"} 2024-11-15T21:54:43,174 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731707682333.2c83200b2c422973a68afce1173aae30.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731707683167"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731707683167"}]},"ts":"1731707683167"} 2024-11-15T21:54:43,174 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731707682333.3f13d1331edcf64c426c3bf9433330e4.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731707683167"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731707683167"}]},"ts":"1731707683167"} 2024-11-15T21:54:43,175 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731707682333.f9ef867a298d175eca9d85304f2faa69.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731707683167"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731707683167"}]},"ts":"1731707683167"} 2024-11-15T21:54:43,175 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731707682333.0890aeec1f0ca0649e6483a2d56f36b4.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731707683167"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731707683167"}]},"ts":"1731707683167"} 2024-11-15T21:54:43,175 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731707682333.d28ee8a2ac06d2199c72ac7f884ba470.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731707683167"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731707683167"}]},"ts":"1731707683167"} 2024-11-15T21:54:43,175 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731707682333.aec2e46cb64898ba2a66825d3732f6dc.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731707683167"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731707683167"}]},"ts":"1731707683167"} 2024-11-15T21:54:43,176 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731707682333.cbb6b5b8341f7ff2ada4ac1931a1a588.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731707683167"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731707683167"}]},"ts":"1731707683167"} 2024-11-15T21:54:43,176 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731707682333.a3f7be24d811f4d6177c28e08f25af2d.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731707683167"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731707683167"}]},"ts":"1731707683167"} 2024-11-15T21:54:43,176 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731707682333.6e3b749fcbfdadfe59f1d1cf94f5406a.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731707683167"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731707683167"}]},"ts":"1731707683167"} 2024-11-15T21:54:43,176 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731707682333.51af260ba7a9b40a7f788a5c93a416b6.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731707683167"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731707683167"}]},"ts":"1731707683167"} 2024-11-15T21:54:43,176 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731707682333.0ab2d104e772f41ee9c56851adad43f0.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731707683167"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731707683167"}]},"ts":"1731707683167"} 2024-11-15T21:54:43,177 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731707682333.3cf264afea186a5cc96edd3e68a10d45.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731707683167"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731707683167"}]},"ts":"1731707683167"} 2024-11-15T21:54:43,177 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731707682333.d100add17d87270d000a3c39b49f8227.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731707683167"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731707683167"}]},"ts":"1731707683167"} 2024-11-15T21:54:43,177 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731707682333.386f4af1e738f142f3fbde37773d4d2c.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731707683167"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731707683167"}]},"ts":"1731707683167"} 2024-11-15T21:54:43,177 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731707682333.d6063c53955b74f03cc36bd578a6bd41.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731707683167"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731707683167"}]},"ts":"1731707683167"} 2024-11-15T21:54:43,177 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731707682333.fac6af87aa0fc9826df0515f766c031e.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731707683167"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731707683167"}]},"ts":"1731707683167"} 2024-11-15T21:54:43,177 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731707682333.5ce283f02f0974e39c98759bb2bc9883.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731707683167"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731707683167"}]},"ts":"1731707683167"} 2024-11-15T21:54:43,178 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731707682333.6694325f9c862a5a1ee02eabf7861cc3.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731707683167"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731707683167"}]},"ts":"1731707683167"} 2024-11-15T21:54:43,178 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731707682333.46b5f83abe9585829b26884a1b26b43f.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731707683167"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731707683167"}]},"ts":"1731707683167"} 2024-11-15T21:54:43,178 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731707682333.e76ae188febdca91d319999f619d9e11.","families":{"info":[{"qualifier":"regioninfo","vlen":106,"tag":[],"timestamp":"1731707683167"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731707683167"}]},"ts":"1731707683167"} 2024-11-15T21:54:43,178 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731707682333.d9989b0341ef96818e811f874dbcf91c.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731707683167"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731707683167"}]},"ts":"1731707683167"} 2024-11-15T21:54:43,178 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731707682333.1031049c0b4149d1189db33e38ab56cd.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731707683167"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731707683167"}]},"ts":"1731707683167"} 2024-11-15T21:54:43,241 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-11-15T21:54:43,244 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T21:54:43,247 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731707683244"}]},"ts":"1731707683244"} 2024-11-15T21:54:43,252 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, state=ENABLING in hbase:meta 2024-11-15T21:54:43,253 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {ae3311ea5ccb=0} racks are {/default-rack=0} 2024-11-15T21:54:43,311 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-15T21:54:43,311 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-15T21:54:43,311 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-15T21:54:43,311 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-15T21:54:43,311 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-15T21:54:43,311 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-15T21:54:43,311 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-15T21:54:43,311 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-15T21:54:43,311 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-15T21:54:43,311 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-15T21:54:43,314 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=229a036fd38ae13e7359b15bc2f2e587, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=dd723478fcfc0e25b606965c9135244b, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=50df10b3ae47c5dc7c4521c4c58c192f, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4256fff2fa33299a6264dd460a4003fb, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2c83200b2c422973a68afce1173aae30, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3f13d1331edcf64c426c3bf9433330e4, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=f9ef867a298d175eca9d85304f2faa69, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=0890aeec1f0ca0649e6483a2d56f36b4, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d28ee8a2ac06d2199c72ac7f884ba470, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=aec2e46cb64898ba2a66825d3732f6dc, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=cbb6b5b8341f7ff2ada4ac1931a1a588, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a3f7be24d811f4d6177c28e08f25af2d, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=6e3b749fcbfdadfe59f1d1cf94f5406a, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=51af260ba7a9b40a7f788a5c93a416b6, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=0ab2d104e772f41ee9c56851adad43f0, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3cf264afea186a5cc96edd3e68a10d45, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d100add17d87270d000a3c39b49f8227, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d9989b0341ef96818e811f874dbcf91c, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=386f4af1e738f142f3fbde37773d4d2c, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d6063c53955b74f03cc36bd578a6bd41, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fac6af87aa0fc9826df0515f766c031e, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5ce283f02f0974e39c98759bb2bc9883, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=6694325f9c862a5a1ee02eabf7861cc3, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=46b5f83abe9585829b26884a1b26b43f, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1031049c0b4149d1189db33e38ab56cd, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e76ae188febdca91d319999f619d9e11, ASSIGN}] 2024-11-15T21:54:43,334 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e76ae188febdca91d319999f619d9e11, ASSIGN 2024-11-15T21:54:43,338 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e76ae188febdca91d319999f619d9e11, ASSIGN; state=OFFLINE, location=ae3311ea5ccb,40941,1731707679516; forceNewPlan=false, retain=false 2024-11-15T21:54:43,342 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1031049c0b4149d1189db33e38ab56cd, ASSIGN 2024-11-15T21:54:43,343 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1031049c0b4149d1189db33e38ab56cd, ASSIGN; state=OFFLINE, location=ae3311ea5ccb,37455,1731707679725; forceNewPlan=false, retain=false 2024-11-15T21:54:43,346 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=46b5f83abe9585829b26884a1b26b43f, ASSIGN 2024-11-15T21:54:43,348 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=46b5f83abe9585829b26884a1b26b43f, ASSIGN; state=OFFLINE, location=ae3311ea5ccb,37455,1731707679725; forceNewPlan=false, retain=false 2024-11-15T21:54:43,350 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=6694325f9c862a5a1ee02eabf7861cc3, ASSIGN 2024-11-15T21:54:43,362 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=6694325f9c862a5a1ee02eabf7861cc3, ASSIGN; state=OFFLINE, location=ae3311ea5ccb,32871,1731707679650; forceNewPlan=false, retain=false 2024-11-15T21:54:43,365 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5ce283f02f0974e39c98759bb2bc9883, ASSIGN 2024-11-15T21:54:43,368 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5ce283f02f0974e39c98759bb2bc9883, ASSIGN; state=OFFLINE, location=ae3311ea5ccb,40941,1731707679516; forceNewPlan=false, retain=false 2024-11-15T21:54:43,370 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fac6af87aa0fc9826df0515f766c031e, ASSIGN 2024-11-15T21:54:43,371 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fac6af87aa0fc9826df0515f766c031e, ASSIGN; state=OFFLINE, location=ae3311ea5ccb,37455,1731707679725; forceNewPlan=false, retain=false 2024-11-15T21:54:43,376 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d6063c53955b74f03cc36bd578a6bd41, ASSIGN 2024-11-15T21:54:43,378 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d6063c53955b74f03cc36bd578a6bd41, ASSIGN; state=OFFLINE, location=ae3311ea5ccb,32871,1731707679650; forceNewPlan=false, retain=false 2024-11-15T21:54:43,379 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=386f4af1e738f142f3fbde37773d4d2c, ASSIGN 2024-11-15T21:54:43,382 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=386f4af1e738f142f3fbde37773d4d2c, ASSIGN; state=OFFLINE, location=ae3311ea5ccb,40941,1731707679516; forceNewPlan=false, retain=false 2024-11-15T21:54:43,384 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d9989b0341ef96818e811f874dbcf91c, ASSIGN 2024-11-15T21:54:43,386 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d9989b0341ef96818e811f874dbcf91c, ASSIGN; state=OFFLINE, location=ae3311ea5ccb,32871,1731707679650; forceNewPlan=false, retain=false 2024-11-15T21:54:43,390 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d100add17d87270d000a3c39b49f8227, ASSIGN 2024-11-15T21:54:43,392 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d100add17d87270d000a3c39b49f8227, ASSIGN; state=OFFLINE, location=ae3311ea5ccb,37455,1731707679725; forceNewPlan=false, retain=false 2024-11-15T21:54:43,394 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3cf264afea186a5cc96edd3e68a10d45, ASSIGN 2024-11-15T21:54:43,395 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3cf264afea186a5cc96edd3e68a10d45, ASSIGN; state=OFFLINE, location=ae3311ea5ccb,32871,1731707679650; forceNewPlan=false, retain=false 2024-11-15T21:54:43,397 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=0ab2d104e772f41ee9c56851adad43f0, ASSIGN 2024-11-15T21:54:43,399 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=0ab2d104e772f41ee9c56851adad43f0, ASSIGN; state=OFFLINE, location=ae3311ea5ccb,40941,1731707679516; forceNewPlan=false, retain=false 2024-11-15T21:54:43,400 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=51af260ba7a9b40a7f788a5c93a416b6, ASSIGN 2024-11-15T21:54:43,403 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=51af260ba7a9b40a7f788a5c93a416b6, ASSIGN; state=OFFLINE, location=ae3311ea5ccb,37455,1731707679725; forceNewPlan=false, retain=false 2024-11-15T21:54:43,404 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=6e3b749fcbfdadfe59f1d1cf94f5406a, ASSIGN 2024-11-15T21:54:43,406 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=6e3b749fcbfdadfe59f1d1cf94f5406a, ASSIGN; state=OFFLINE, location=ae3311ea5ccb,32871,1731707679650; forceNewPlan=false, retain=false 2024-11-15T21:54:43,408 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a3f7be24d811f4d6177c28e08f25af2d, ASSIGN 2024-11-15T21:54:43,413 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a3f7be24d811f4d6177c28e08f25af2d, ASSIGN; state=OFFLINE, location=ae3311ea5ccb,40941,1731707679516; forceNewPlan=false, retain=false 2024-11-15T21:54:43,415 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=cbb6b5b8341f7ff2ada4ac1931a1a588, ASSIGN 2024-11-15T21:54:43,416 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=cbb6b5b8341f7ff2ada4ac1931a1a588, ASSIGN; state=OFFLINE, location=ae3311ea5ccb,37455,1731707679725; forceNewPlan=false, retain=false 2024-11-15T21:54:43,417 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=aec2e46cb64898ba2a66825d3732f6dc, ASSIGN 2024-11-15T21:54:43,418 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=aec2e46cb64898ba2a66825d3732f6dc, ASSIGN; state=OFFLINE, location=ae3311ea5ccb,32871,1731707679650; forceNewPlan=false, retain=false 2024-11-15T21:54:43,419 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d28ee8a2ac06d2199c72ac7f884ba470, ASSIGN 2024-11-15T21:54:43,421 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d28ee8a2ac06d2199c72ac7f884ba470, ASSIGN; state=OFFLINE, location=ae3311ea5ccb,40941,1731707679516; forceNewPlan=false, retain=false 2024-11-15T21:54:43,422 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=0890aeec1f0ca0649e6483a2d56f36b4, ASSIGN 2024-11-15T21:54:43,424 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=0890aeec1f0ca0649e6483a2d56f36b4, ASSIGN; state=OFFLINE, location=ae3311ea5ccb,37455,1731707679725; forceNewPlan=false, retain=false 2024-11-15T21:54:43,427 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=f9ef867a298d175eca9d85304f2faa69, ASSIGN 2024-11-15T21:54:43,429 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=f9ef867a298d175eca9d85304f2faa69, ASSIGN; state=OFFLINE, location=ae3311ea5ccb,32871,1731707679650; forceNewPlan=false, retain=false 2024-11-15T21:54:43,430 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3f13d1331edcf64c426c3bf9433330e4, ASSIGN 2024-11-15T21:54:43,432 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3f13d1331edcf64c426c3bf9433330e4, ASSIGN; state=OFFLINE, location=ae3311ea5ccb,40941,1731707679516; forceNewPlan=false, retain=false 2024-11-15T21:54:43,434 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2c83200b2c422973a68afce1173aae30, ASSIGN 2024-11-15T21:54:43,435 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2c83200b2c422973a68afce1173aae30, ASSIGN; state=OFFLINE, location=ae3311ea5ccb,37455,1731707679725; forceNewPlan=false, retain=false 2024-11-15T21:54:43,438 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4256fff2fa33299a6264dd460a4003fb, ASSIGN 2024-11-15T21:54:43,439 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4256fff2fa33299a6264dd460a4003fb, ASSIGN; state=OFFLINE, location=ae3311ea5ccb,32871,1731707679650; forceNewPlan=false, retain=false 2024-11-15T21:54:43,441 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=50df10b3ae47c5dc7c4521c4c58c192f, ASSIGN 2024-11-15T21:54:43,444 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=50df10b3ae47c5dc7c4521c4c58c192f, ASSIGN; state=OFFLINE, location=ae3311ea5ccb,37455,1731707679725; forceNewPlan=false, retain=false 2024-11-15T21:54:43,447 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=dd723478fcfc0e25b606965c9135244b, ASSIGN 2024-11-15T21:54:43,448 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=dd723478fcfc0e25b606965c9135244b, ASSIGN; state=OFFLINE, location=ae3311ea5ccb,32871,1731707679650; forceNewPlan=false, retain=false 2024-11-15T21:54:43,450 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=229a036fd38ae13e7359b15bc2f2e587, ASSIGN 2024-11-15T21:54:43,451 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=229a036fd38ae13e7359b15bc2f2e587, ASSIGN; state=OFFLINE, location=ae3311ea5ccb,40941,1731707679516; forceNewPlan=false, retain=false 2024-11-15T21:54:43,492 INFO [ae3311ea5ccb:36531 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-11-15T21:54:43,493 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=e76ae188febdca91d319999f619d9e11, regionState=OPENING, regionLocation=ae3311ea5ccb,40941,1731707679516 2024-11-15T21:54:43,494 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=5ce283f02f0974e39c98759bb2bc9883, regionState=OPENING, regionLocation=ae3311ea5ccb,40941,1731707679516 2024-11-15T21:54:43,495 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=386f4af1e738f142f3fbde37773d4d2c, regionState=OPENING, regionLocation=ae3311ea5ccb,40941,1731707679516 2024-11-15T21:54:43,496 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=0ab2d104e772f41ee9c56851adad43f0, regionState=OPENING, regionLocation=ae3311ea5ccb,40941,1731707679516 2024-11-15T21:54:43,496 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=a3f7be24d811f4d6177c28e08f25af2d, regionState=OPENING, regionLocation=ae3311ea5ccb,40941,1731707679516 2024-11-15T21:54:43,497 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=d28ee8a2ac06d2199c72ac7f884ba470, regionState=OPENING, regionLocation=ae3311ea5ccb,40941,1731707679516 2024-11-15T21:54:43,497 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=3f13d1331edcf64c426c3bf9433330e4, regionState=OPENING, regionLocation=ae3311ea5ccb,40941,1731707679516 2024-11-15T21:54:43,498 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=229a036fd38ae13e7359b15bc2f2e587, regionState=OPENING, regionLocation=ae3311ea5ccb,40941,1731707679516 2024-11-15T21:54:43,498 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e76ae188febdca91d319999f619d9e11, ASSIGN because future has completed 2024-11-15T21:54:43,498 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=1031049c0b4149d1189db33e38ab56cd, regionState=OPENING, regionLocation=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:54:43,499 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=46b5f83abe9585829b26884a1b26b43f, regionState=OPENING, regionLocation=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:54:43,499 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=fac6af87aa0fc9826df0515f766c031e, regionState=OPENING, regionLocation=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:54:43,500 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=d100add17d87270d000a3c39b49f8227, regionState=OPENING, regionLocation=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:54:43,500 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=51af260ba7a9b40a7f788a5c93a416b6, regionState=OPENING, regionLocation=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:54:43,501 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=cbb6b5b8341f7ff2ada4ac1931a1a588, regionState=OPENING, regionLocation=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:54:43,501 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=0890aeec1f0ca0649e6483a2d56f36b4, regionState=OPENING, regionLocation=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:54:43,502 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure e76ae188febdca91d319999f619d9e11, server=ae3311ea5ccb,40941,1731707679516}] 2024-11-15T21:54:43,505 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5ce283f02f0974e39c98759bb2bc9883, ASSIGN because future has completed 2024-11-15T21:54:43,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T21:54:43,507 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=386f4af1e738f142f3fbde37773d4d2c, ASSIGN because future has completed 2024-11-15T21:54:43,510 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=0ab2d104e772f41ee9c56851adad43f0, ASSIGN because future has completed 2024-11-15T21:54:43,515 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a3f7be24d811f4d6177c28e08f25af2d, ASSIGN because future has completed 2024-11-15T21:54:43,516 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0ab2d104e772f41ee9c56851adad43f0, server=ae3311ea5ccb,40941,1731707679516}] 2024-11-15T21:54:43,518 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d28ee8a2ac06d2199c72ac7f884ba470, ASSIGN because future has completed 2024-11-15T21:54:43,521 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3f13d1331edcf64c426c3bf9433330e4, ASSIGN because future has completed 2024-11-15T21:54:43,525 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=229a036fd38ae13e7359b15bc2f2e587, ASSIGN because future has completed 2024-11-15T21:54:43,525 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3f13d1331edcf64c426c3bf9433330e4, server=ae3311ea5ccb,40941,1731707679516}] 2024-11-15T21:54:43,527 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1031049c0b4149d1189db33e38ab56cd, ASSIGN because future has completed 2024-11-15T21:54:43,528 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=46b5f83abe9585829b26884a1b26b43f, ASSIGN because future has completed 2024-11-15T21:54:43,531 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fac6af87aa0fc9826df0515f766c031e, ASSIGN because future has completed 2024-11-15T21:54:43,532 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure fac6af87aa0fc9826df0515f766c031e, server=ae3311ea5ccb,37455,1731707679725}] 2024-11-15T21:54:43,532 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d100add17d87270d000a3c39b49f8227, ASSIGN because future has completed 2024-11-15T21:54:43,534 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=51af260ba7a9b40a7f788a5c93a416b6, ASSIGN because future has completed 2024-11-15T21:54:43,536 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=cbb6b5b8341f7ff2ada4ac1931a1a588, ASSIGN because future has completed 2024-11-15T21:54:43,537 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=0890aeec1f0ca0649e6483a2d56f36b4, ASSIGN because future has completed 2024-11-15T21:54:43,538 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0890aeec1f0ca0649e6483a2d56f36b4, server=ae3311ea5ccb,37455,1731707679725}] 2024-11-15T21:54:43,543 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure cbb6b5b8341f7ff2ada4ac1931a1a588, server=ae3311ea5ccb,37455,1731707679725}] 2024-11-15T21:54:43,550 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 51af260ba7a9b40a7f788a5c93a416b6, server=ae3311ea5ccb,37455,1731707679725}] 2024-11-15T21:54:43,556 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure d100add17d87270d000a3c39b49f8227, server=ae3311ea5ccb,37455,1731707679725}] 2024-11-15T21:54:43,561 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 46b5f83abe9585829b26884a1b26b43f, server=ae3311ea5ccb,37455,1731707679725}] 2024-11-15T21:54:43,568 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1031049c0b4149d1189db33e38ab56cd, server=ae3311ea5ccb,37455,1731707679725}] 2024-11-15T21:54:43,574 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 229a036fd38ae13e7359b15bc2f2e587, server=ae3311ea5ccb,40941,1731707679516}] 2024-11-15T21:54:43,579 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure d28ee8a2ac06d2199c72ac7f884ba470, server=ae3311ea5ccb,40941,1731707679516}] 2024-11-15T21:54:43,583 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure a3f7be24d811f4d6177c28e08f25af2d, server=ae3311ea5ccb,40941,1731707679516}] 2024-11-15T21:54:43,588 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 386f4af1e738f142f3fbde37773d4d2c, server=ae3311ea5ccb,40941,1731707679516}] 2024-11-15T21:54:43,594 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5ce283f02f0974e39c98759bb2bc9883, server=ae3311ea5ccb,40941,1731707679516}] 2024-11-15T21:54:43,600 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=2c83200b2c422973a68afce1173aae30, regionState=OPENING, regionLocation=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:54:43,601 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=50df10b3ae47c5dc7c4521c4c58c192f, regionState=OPENING, regionLocation=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:54:43,601 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=6694325f9c862a5a1ee02eabf7861cc3, regionState=OPENING, regionLocation=ae3311ea5ccb,32871,1731707679650 2024-11-15T21:54:43,602 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=d6063c53955b74f03cc36bd578a6bd41, regionState=OPENING, regionLocation=ae3311ea5ccb,32871,1731707679650 2024-11-15T21:54:43,603 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=d9989b0341ef96818e811f874dbcf91c, regionState=OPENING, regionLocation=ae3311ea5ccb,32871,1731707679650 2024-11-15T21:54:43,603 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=3cf264afea186a5cc96edd3e68a10d45, regionState=OPENING, regionLocation=ae3311ea5ccb,32871,1731707679650 2024-11-15T21:54:43,604 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2c83200b2c422973a68afce1173aae30, ASSIGN because future has completed 2024-11-15T21:54:43,604 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2c83200b2c422973a68afce1173aae30, server=ae3311ea5ccb,37455,1731707679725}] 2024-11-15T21:54:43,605 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=50df10b3ae47c5dc7c4521c4c58c192f, ASSIGN because future has completed 2024-11-15T21:54:43,607 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=6694325f9c862a5a1ee02eabf7861cc3, ASSIGN because future has completed 2024-11-15T21:54:43,609 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d6063c53955b74f03cc36bd578a6bd41, ASSIGN because future has completed 2024-11-15T21:54:43,611 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d9989b0341ef96818e811f874dbcf91c, ASSIGN because future has completed 2024-11-15T21:54:43,612 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3cf264afea186a5cc96edd3e68a10d45, ASSIGN because future has completed 2024-11-15T21:54:43,614 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3cf264afea186a5cc96edd3e68a10d45, server=ae3311ea5ccb,32871,1731707679650}] 2024-11-15T21:54:43,618 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure d9989b0341ef96818e811f874dbcf91c, server=ae3311ea5ccb,32871,1731707679650}] 2024-11-15T21:54:43,623 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure d6063c53955b74f03cc36bd578a6bd41, server=ae3311ea5ccb,32871,1731707679650}] 2024-11-15T21:54:43,627 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6694325f9c862a5a1ee02eabf7861cc3, server=ae3311ea5ccb,32871,1731707679650}] 2024-11-15T21:54:43,633 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 50df10b3ae47c5dc7c4521c4c58c192f, server=ae3311ea5ccb,37455,1731707679725}] 2024-11-15T21:54:43,640 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=6e3b749fcbfdadfe59f1d1cf94f5406a, regionState=OPENING, regionLocation=ae3311ea5ccb,32871,1731707679650 2024-11-15T21:54:43,641 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=aec2e46cb64898ba2a66825d3732f6dc, regionState=OPENING, regionLocation=ae3311ea5ccb,32871,1731707679650 2024-11-15T21:54:43,642 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=f9ef867a298d175eca9d85304f2faa69, regionState=OPENING, regionLocation=ae3311ea5ccb,32871,1731707679650 2024-11-15T21:54:43,643 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=4256fff2fa33299a6264dd460a4003fb, regionState=OPENING, regionLocation=ae3311ea5ccb,32871,1731707679650 2024-11-15T21:54:43,643 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=6e3b749fcbfdadfe59f1d1cf94f5406a, ASSIGN because future has completed 2024-11-15T21:54:43,644 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6e3b749fcbfdadfe59f1d1cf94f5406a, server=ae3311ea5ccb,32871,1731707679650}] 2024-11-15T21:54:43,645 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=aec2e46cb64898ba2a66825d3732f6dc, ASSIGN because future has completed 2024-11-15T21:54:43,647 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=f9ef867a298d175eca9d85304f2faa69, ASSIGN because future has completed 2024-11-15T21:54:43,648 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4256fff2fa33299a6264dd460a4003fb, ASSIGN because future has completed 2024-11-15T21:54:43,650 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4256fff2fa33299a6264dd460a4003fb, server=ae3311ea5ccb,32871,1731707679650}] 2024-11-15T21:54:43,655 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure f9ef867a298d175eca9d85304f2faa69, server=ae3311ea5ccb,32871,1731707679650}] 2024-11-15T21:54:43,658 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-15T21:54:43,661 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47221, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-15T21:54:43,661 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure aec2e46cb64898ba2a66825d3732f6dc, server=ae3311ea5ccb,32871,1731707679650}] 2024-11-15T21:54:43,673 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=dd723478fcfc0e25b606965c9135244b, regionState=OPENING, regionLocation=ae3311ea5ccb,32871,1731707679650 2024-11-15T21:54:43,673 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731707682333.229a036fd38ae13e7359b15bc2f2e587. 2024-11-15T21:54:43,674 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 229a036fd38ae13e7359b15bc2f2e587, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731707682333.229a036fd38ae13e7359b15bc2f2e587.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-15T21:54:43,674 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 229a036fd38ae13e7359b15bc2f2e587 2024-11-15T21:54:43,675 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731707682333.229a036fd38ae13e7359b15bc2f2e587.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:43,675 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 229a036fd38ae13e7359b15bc2f2e587 2024-11-15T21:54:43,675 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 229a036fd38ae13e7359b15bc2f2e587 2024-11-15T21:54:43,677 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=dd723478fcfc0e25b606965c9135244b, ASSIGN because future has completed 2024-11-15T21:54:43,677 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure dd723478fcfc0e25b606965c9135244b, server=ae3311ea5ccb,32871,1731707679650}] 2024-11-15T21:54:43,681 INFO [StoreOpener-229a036fd38ae13e7359b15bc2f2e587-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 229a036fd38ae13e7359b15bc2f2e587 2024-11-15T21:54:43,684 INFO [StoreOpener-229a036fd38ae13e7359b15bc2f2e587-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 229a036fd38ae13e7359b15bc2f2e587 columnFamilyName colfamily11 2024-11-15T21:54:43,684 DEBUG [StoreOpener-229a036fd38ae13e7359b15bc2f2e587-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:43,685 INFO [StoreOpener-229a036fd38ae13e7359b15bc2f2e587-1 {}] regionserver.HStore(327): Store=229a036fd38ae13e7359b15bc2f2e587/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:43,685 INFO [StoreOpener-229a036fd38ae13e7359b15bc2f2e587-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 229a036fd38ae13e7359b15bc2f2e587 2024-11-15T21:54:43,686 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-15T21:54:43,688 INFO [StoreOpener-229a036fd38ae13e7359b15bc2f2e587-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 229a036fd38ae13e7359b15bc2f2e587 columnFamilyName colfamily21 2024-11-15T21:54:43,688 DEBUG [StoreOpener-229a036fd38ae13e7359b15bc2f2e587-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:43,688 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35575, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-15T21:54:43,689 INFO [StoreOpener-229a036fd38ae13e7359b15bc2f2e587-1 {}] regionserver.HStore(327): Store=229a036fd38ae13e7359b15bc2f2e587/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:43,689 INFO [StoreOpener-229a036fd38ae13e7359b15bc2f2e587-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 229a036fd38ae13e7359b15bc2f2e587 2024-11-15T21:54:43,691 INFO [StoreOpener-229a036fd38ae13e7359b15bc2f2e587-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 229a036fd38ae13e7359b15bc2f2e587 columnFamilyName colfamily31 2024-11-15T21:54:43,691 DEBUG [StoreOpener-229a036fd38ae13e7359b15bc2f2e587-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:43,692 INFO [StoreOpener-229a036fd38ae13e7359b15bc2f2e587-1 {}] regionserver.HStore(327): Store=229a036fd38ae13e7359b15bc2f2e587/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:43,692 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 229a036fd38ae13e7359b15bc2f2e587 2024-11-15T21:54:43,694 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/229a036fd38ae13e7359b15bc2f2e587 2024-11-15T21:54:43,695 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/229a036fd38ae13e7359b15bc2f2e587 2024-11-15T21:54:43,696 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 229a036fd38ae13e7359b15bc2f2e587 2024-11-15T21:54:43,697 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 229a036fd38ae13e7359b15bc2f2e587 2024-11-15T21:54:43,697 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731707682333.d100add17d87270d000a3c39b49f8227. 2024-11-15T21:54:43,697 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-15T21:54:43,698 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => d100add17d87270d000a3c39b49f8227, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731707682333.d100add17d87270d000a3c39b49f8227.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-15T21:54:43,698 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true d100add17d87270d000a3c39b49f8227 2024-11-15T21:54:43,698 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731707682333.d100add17d87270d000a3c39b49f8227.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:43,698 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for d100add17d87270d000a3c39b49f8227 2024-11-15T21:54:43,698 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for d100add17d87270d000a3c39b49f8227 2024-11-15T21:54:43,700 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 229a036fd38ae13e7359b15bc2f2e587 2024-11-15T21:54:43,701 INFO [StoreOpener-d100add17d87270d000a3c39b49f8227-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region d100add17d87270d000a3c39b49f8227 2024-11-15T21:54:43,704 INFO [StoreOpener-d100add17d87270d000a3c39b49f8227-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d100add17d87270d000a3c39b49f8227 columnFamilyName colfamily11 2024-11-15T21:54:43,705 DEBUG [StoreOpener-d100add17d87270d000a3c39b49f8227-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:43,705 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/229a036fd38ae13e7359b15bc2f2e587/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T21:54:43,705 INFO [StoreOpener-d100add17d87270d000a3c39b49f8227-1 {}] regionserver.HStore(327): Store=d100add17d87270d000a3c39b49f8227/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:43,706 INFO [StoreOpener-d100add17d87270d000a3c39b49f8227-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region d100add17d87270d000a3c39b49f8227 2024-11-15T21:54:43,706 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 229a036fd38ae13e7359b15bc2f2e587; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58797808, jitterRate=-0.12384438514709473}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-15T21:54:43,706 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 229a036fd38ae13e7359b15bc2f2e587 2024-11-15T21:54:43,707 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 229a036fd38ae13e7359b15bc2f2e587: Running coprocessor pre-open hook at 1731707683675Writing region info on filesystem at 1731707683675Initializing all the Stores at 1731707683677 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707683677Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707683681 (+4 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707683681Cleaning up temporary data from old regions at 1731707683697 (+16 ms)Running coprocessor post-open hooks at 1731707683706 (+9 ms)Region opened successfully at 1731707683707 (+1 ms) 2024-11-15T21:54:43,708 INFO [StoreOpener-d100add17d87270d000a3c39b49f8227-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d100add17d87270d000a3c39b49f8227 columnFamilyName colfamily21 2024-11-15T21:54:43,708 DEBUG [StoreOpener-d100add17d87270d000a3c39b49f8227-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:43,709 INFO [StoreOpener-d100add17d87270d000a3c39b49f8227-1 {}] regionserver.HStore(327): Store=d100add17d87270d000a3c39b49f8227/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:43,709 INFO [StoreOpener-d100add17d87270d000a3c39b49f8227-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region d100add17d87270d000a3c39b49f8227 2024-11-15T21:54:43,711 INFO [StoreOpener-d100add17d87270d000a3c39b49f8227-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d100add17d87270d000a3c39b49f8227 columnFamilyName colfamily31 2024-11-15T21:54:43,711 DEBUG [StoreOpener-d100add17d87270d000a3c39b49f8227-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:43,712 INFO [StoreOpener-d100add17d87270d000a3c39b49f8227-1 {}] regionserver.HStore(327): Store=d100add17d87270d000a3c39b49f8227/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:43,712 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for d100add17d87270d000a3c39b49f8227 2024-11-15T21:54:43,714 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d100add17d87270d000a3c39b49f8227 2024-11-15T21:54:43,714 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731707682333.229a036fd38ae13e7359b15bc2f2e587., pid=41, masterSystemTime=1731707683657 2024-11-15T21:54:43,715 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d100add17d87270d000a3c39b49f8227 2024-11-15T21:54:43,717 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for d100add17d87270d000a3c39b49f8227 2024-11-15T21:54:43,717 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for d100add17d87270d000a3c39b49f8227 2024-11-15T21:54:43,718 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731707682333.229a036fd38ae13e7359b15bc2f2e587. 2024-11-15T21:54:43,718 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731707682333.229a036fd38ae13e7359b15bc2f2e587. 2024-11-15T21:54:43,718 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-15T21:54:43,718 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731707682333.5ce283f02f0974e39c98759bb2bc9883. 2024-11-15T21:54:43,718 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => 5ce283f02f0974e39c98759bb2bc9883, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731707682333.5ce283f02f0974e39c98759bb2bc9883.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-15T21:54:43,719 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=229a036fd38ae13e7359b15bc2f2e587, regionState=OPEN, openSeqNum=2, regionLocation=ae3311ea5ccb,40941,1731707679516 2024-11-15T21:54:43,719 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 5ce283f02f0974e39c98759bb2bc9883 2024-11-15T21:54:43,720 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731707682333.5ce283f02f0974e39c98759bb2bc9883.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:43,720 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for 5ce283f02f0974e39c98759bb2bc9883 2024-11-15T21:54:43,720 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for d100add17d87270d000a3c39b49f8227 2024-11-15T21:54:43,721 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for 5ce283f02f0974e39c98759bb2bc9883 2024-11-15T21:54:43,724 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 229a036fd38ae13e7359b15bc2f2e587, server=ae3311ea5ccb,40941,1731707679516 because future has completed 2024-11-15T21:54:43,728 INFO [StoreOpener-5ce283f02f0974e39c98759bb2bc9883-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 5ce283f02f0974e39c98759bb2bc9883 2024-11-15T21:54:43,729 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d100add17d87270d000a3c39b49f8227/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T21:54:43,730 INFO [StoreOpener-5ce283f02f0974e39c98759bb2bc9883-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5ce283f02f0974e39c98759bb2bc9883 columnFamilyName colfamily11 2024-11-15T21:54:43,730 DEBUG [StoreOpener-5ce283f02f0974e39c98759bb2bc9883-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:43,731 INFO [StoreOpener-5ce283f02f0974e39c98759bb2bc9883-1 {}] regionserver.HStore(327): Store=5ce283f02f0974e39c98759bb2bc9883/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:43,731 INFO [StoreOpener-5ce283f02f0974e39c98759bb2bc9883-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 5ce283f02f0974e39c98759bb2bc9883 2024-11-15T21:54:43,733 INFO [StoreOpener-5ce283f02f0974e39c98759bb2bc9883-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5ce283f02f0974e39c98759bb2bc9883 columnFamilyName colfamily21 2024-11-15T21:54:43,733 DEBUG [StoreOpener-5ce283f02f0974e39c98759bb2bc9883-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:43,733 INFO [StoreOpener-5ce283f02f0974e39c98759bb2bc9883-1 {}] regionserver.HStore(327): Store=5ce283f02f0974e39c98759bb2bc9883/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:43,734 INFO [StoreOpener-5ce283f02f0974e39c98759bb2bc9883-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 5ce283f02f0974e39c98759bb2bc9883 2024-11-15T21:54:43,734 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened d100add17d87270d000a3c39b49f8227; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60322966, jitterRate=-0.10111775994300842}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-15T21:54:43,735 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d100add17d87270d000a3c39b49f8227 2024-11-15T21:54:43,735 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for d100add17d87270d000a3c39b49f8227: Running coprocessor pre-open hook at 1731707683699Writing region info on filesystem at 1731707683699Initializing all the Stores at 1731707683700 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707683700Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707683700Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707683700Cleaning up temporary data from old regions at 1731707683717 (+17 ms)Running coprocessor post-open hooks at 1731707683735 (+18 ms)Region opened successfully at 1731707683735 2024-11-15T21:54:43,736 INFO [StoreOpener-5ce283f02f0974e39c98759bb2bc9883-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5ce283f02f0974e39c98759bb2bc9883 columnFamilyName colfamily31 2024-11-15T21:54:43,736 DEBUG [StoreOpener-5ce283f02f0974e39c98759bb2bc9883-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:43,737 INFO [StoreOpener-5ce283f02f0974e39c98759bb2bc9883-1 {}] regionserver.HStore(327): Store=5ce283f02f0974e39c98759bb2bc9883/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:43,737 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for 5ce283f02f0974e39c98759bb2bc9883 2024-11-15T21:54:43,739 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731707682333.d100add17d87270d000a3c39b49f8227., pid=38, masterSystemTime=1731707683686 2024-11-15T21:54:43,739 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ce283f02f0974e39c98759bb2bc9883 2024-11-15T21:54:43,740 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ce283f02f0974e39c98759bb2bc9883 2024-11-15T21:54:43,742 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for 5ce283f02f0974e39c98759bb2bc9883 2024-11-15T21:54:43,742 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for 5ce283f02f0974e39c98759bb2bc9883 2024-11-15T21:54:43,744 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-15T21:54:43,749 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for 5ce283f02f0974e39c98759bb2bc9883 2024-11-15T21:54:43,750 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=5 2024-11-15T21:54:43,750 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 229a036fd38ae13e7359b15bc2f2e587, server=ae3311ea5ccb,40941,1731707679516 in 160 msec 2024-11-15T21:54:43,752 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=d100add17d87270d000a3c39b49f8227, regionState=OPEN, openSeqNum=2, regionLocation=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:54:43,754 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731707682333.d100add17d87270d000a3c39b49f8227. 2024-11-15T21:54:43,754 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731707682333.d100add17d87270d000a3c39b49f8227. 2024-11-15T21:54:43,754 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731707682333.46b5f83abe9585829b26884a1b26b43f. 2024-11-15T21:54:43,755 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 46b5f83abe9585829b26884a1b26b43f, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731707682333.46b5f83abe9585829b26884a1b26b43f.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-15T21:54:43,755 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 46b5f83abe9585829b26884a1b26b43f 2024-11-15T21:54:43,755 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731707682333.46b5f83abe9585829b26884a1b26b43f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:43,755 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 46b5f83abe9585829b26884a1b26b43f 2024-11-15T21:54:43,756 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 46b5f83abe9585829b26884a1b26b43f 2024-11-15T21:54:43,756 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ce283f02f0974e39c98759bb2bc9883/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T21:54:43,757 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened 5ce283f02f0974e39c98759bb2bc9883; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70785241, jitterRate=0.054782286286354065}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-15T21:54:43,758 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5ce283f02f0974e39c98759bb2bc9883 2024-11-15T21:54:43,758 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for 5ce283f02f0974e39c98759bb2bc9883: Running coprocessor pre-open hook at 1731707683721Writing region info on filesystem at 1731707683721Initializing all the Stores at 1731707683723 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707683723Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707683728 (+5 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707683728Cleaning up temporary data from old regions at 1731707683742 (+14 ms)Running coprocessor post-open hooks at 1731707683758 (+16 ms)Region opened successfully at 1731707683758 2024-11-15T21:54:43,759 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=229a036fd38ae13e7359b15bc2f2e587, ASSIGN in 440 msec 2024-11-15T21:54:43,761 INFO [StoreOpener-46b5f83abe9585829b26884a1b26b43f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 46b5f83abe9585829b26884a1b26b43f 2024-11-15T21:54:43,762 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731707682333.5ce283f02f0974e39c98759bb2bc9883., pid=45, masterSystemTime=1731707683657 2024-11-15T21:54:43,763 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure d100add17d87270d000a3c39b49f8227, server=ae3311ea5ccb,37455,1731707679725 because future has completed 2024-11-15T21:54:43,764 INFO [StoreOpener-46b5f83abe9585829b26884a1b26b43f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 46b5f83abe9585829b26884a1b26b43f columnFamilyName colfamily11 2024-11-15T21:54:43,764 DEBUG [StoreOpener-46b5f83abe9585829b26884a1b26b43f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:43,765 INFO [StoreOpener-46b5f83abe9585829b26884a1b26b43f-1 {}] regionserver.HStore(327): Store=46b5f83abe9585829b26884a1b26b43f/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:43,766 INFO [StoreOpener-46b5f83abe9585829b26884a1b26b43f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 46b5f83abe9585829b26884a1b26b43f 2024-11-15T21:54:43,768 INFO [StoreOpener-46b5f83abe9585829b26884a1b26b43f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 46b5f83abe9585829b26884a1b26b43f columnFamilyName colfamily21 2024-11-15T21:54:43,768 DEBUG [StoreOpener-46b5f83abe9585829b26884a1b26b43f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:43,770 INFO [StoreOpener-46b5f83abe9585829b26884a1b26b43f-1 {}] regionserver.HStore(327): Store=46b5f83abe9585829b26884a1b26b43f/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:43,770 INFO [StoreOpener-46b5f83abe9585829b26884a1b26b43f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 46b5f83abe9585829b26884a1b26b43f 2024-11-15T21:54:43,773 INFO [StoreOpener-46b5f83abe9585829b26884a1b26b43f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 46b5f83abe9585829b26884a1b26b43f columnFamilyName colfamily31 2024-11-15T21:54:43,773 DEBUG [StoreOpener-46b5f83abe9585829b26884a1b26b43f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:43,774 INFO [StoreOpener-46b5f83abe9585829b26884a1b26b43f-1 {}] regionserver.HStore(327): Store=46b5f83abe9585829b26884a1b26b43f/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:43,774 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731707682333.6e3b749fcbfdadfe59f1d1cf94f5406a. 2024-11-15T21:54:43,774 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 46b5f83abe9585829b26884a1b26b43f 2024-11-15T21:54:43,774 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => 6e3b749fcbfdadfe59f1d1cf94f5406a, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731707682333.6e3b749fcbfdadfe59f1d1cf94f5406a.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-15T21:54:43,774 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 6e3b749fcbfdadfe59f1d1cf94f5406a 2024-11-15T21:54:43,775 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731707682333.6e3b749fcbfdadfe59f1d1cf94f5406a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:43,775 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for 6e3b749fcbfdadfe59f1d1cf94f5406a 2024-11-15T21:54:43,775 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for 6e3b749fcbfdadfe59f1d1cf94f5406a 2024-11-15T21:54:43,775 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/46b5f83abe9585829b26884a1b26b43f 2024-11-15T21:54:43,776 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/46b5f83abe9585829b26884a1b26b43f 2024-11-15T21:54:43,777 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731707682333.5ce283f02f0974e39c98759bb2bc9883. 2024-11-15T21:54:43,777 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731707682333.5ce283f02f0974e39c98759bb2bc9883. 2024-11-15T21:54:43,777 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731707682333.386f4af1e738f142f3fbde37773d4d2c. 2024-11-15T21:54:43,778 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => 386f4af1e738f142f3fbde37773d4d2c, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731707682333.386f4af1e738f142f3fbde37773d4d2c.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-15T21:54:43,778 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 46b5f83abe9585829b26884a1b26b43f 2024-11-15T21:54:43,778 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 46b5f83abe9585829b26884a1b26b43f 2024-11-15T21:54:43,778 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 386f4af1e738f142f3fbde37773d4d2c 2024-11-15T21:54:43,778 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731707682333.386f4af1e738f142f3fbde37773d4d2c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:43,778 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for 386f4af1e738f142f3fbde37773d4d2c 2024-11-15T21:54:43,779 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for 386f4af1e738f142f3fbde37773d4d2c 2024-11-15T21:54:43,779 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-15T21:54:43,782 INFO [StoreOpener-6e3b749fcbfdadfe59f1d1cf94f5406a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 6e3b749fcbfdadfe59f1d1cf94f5406a 2024-11-15T21:54:43,783 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 46b5f83abe9585829b26884a1b26b43f 2024-11-15T21:54:43,784 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=21 2024-11-15T21:54:43,784 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure d100add17d87270d000a3c39b49f8227, server=ae3311ea5ccb,37455,1731707679725 in 208 msec 2024-11-15T21:54:43,784 INFO [StoreOpener-6e3b749fcbfdadfe59f1d1cf94f5406a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6e3b749fcbfdadfe59f1d1cf94f5406a columnFamilyName colfamily11 2024-11-15T21:54:43,784 DEBUG [StoreOpener-6e3b749fcbfdadfe59f1d1cf94f5406a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:43,785 INFO [StoreOpener-6e3b749fcbfdadfe59f1d1cf94f5406a-1 {}] regionserver.HStore(327): Store=6e3b749fcbfdadfe59f1d1cf94f5406a/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:43,786 INFO [StoreOpener-6e3b749fcbfdadfe59f1d1cf94f5406a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 6e3b749fcbfdadfe59f1d1cf94f5406a 2024-11-15T21:54:43,789 INFO [StoreOpener-6e3b749fcbfdadfe59f1d1cf94f5406a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6e3b749fcbfdadfe59f1d1cf94f5406a columnFamilyName colfamily21 2024-11-15T21:54:43,789 DEBUG [StoreOpener-6e3b749fcbfdadfe59f1d1cf94f5406a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:43,790 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d100add17d87270d000a3c39b49f8227, ASSIGN in 470 msec 2024-11-15T21:54:43,791 INFO [StoreOpener-386f4af1e738f142f3fbde37773d4d2c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 386f4af1e738f142f3fbde37773d4d2c 2024-11-15T21:54:43,791 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=5ce283f02f0974e39c98759bb2bc9883, regionState=OPEN, openSeqNum=2, regionLocation=ae3311ea5ccb,40941,1731707679516 2024-11-15T21:54:43,792 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/46b5f83abe9585829b26884a1b26b43f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T21:54:43,793 INFO [StoreOpener-6e3b749fcbfdadfe59f1d1cf94f5406a-1 {}] regionserver.HStore(327): Store=6e3b749fcbfdadfe59f1d1cf94f5406a/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:43,793 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 46b5f83abe9585829b26884a1b26b43f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65663414, jitterRate=-0.021538883447647095}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-15T21:54:43,793 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 46b5f83abe9585829b26884a1b26b43f 2024-11-15T21:54:43,793 INFO [StoreOpener-6e3b749fcbfdadfe59f1d1cf94f5406a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 6e3b749fcbfdadfe59f1d1cf94f5406a 2024-11-15T21:54:43,794 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 46b5f83abe9585829b26884a1b26b43f: Running coprocessor pre-open hook at 1731707683756Writing region info on filesystem at 1731707683756Initializing all the Stores at 1731707683758 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707683758Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707683761 (+3 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707683761Cleaning up temporary data from old regions at 1731707683778 (+17 ms)Running coprocessor post-open hooks at 1731707683793 (+15 ms)Region opened successfully at 1731707683794 (+1 ms) 2024-11-15T21:54:43,795 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5ce283f02f0974e39c98759bb2bc9883, server=ae3311ea5ccb,40941,1731707679516 because future has completed 2024-11-15T21:54:43,796 INFO [StoreOpener-386f4af1e738f142f3fbde37773d4d2c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 386f4af1e738f142f3fbde37773d4d2c columnFamilyName colfamily11 2024-11-15T21:54:43,796 DEBUG [StoreOpener-386f4af1e738f142f3fbde37773d4d2c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:43,796 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731707682333.46b5f83abe9585829b26884a1b26b43f., pid=39, masterSystemTime=1731707683686 2024-11-15T21:54:43,797 INFO [StoreOpener-6e3b749fcbfdadfe59f1d1cf94f5406a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6e3b749fcbfdadfe59f1d1cf94f5406a columnFamilyName colfamily31 2024-11-15T21:54:43,797 DEBUG [StoreOpener-6e3b749fcbfdadfe59f1d1cf94f5406a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:43,799 INFO [StoreOpener-6e3b749fcbfdadfe59f1d1cf94f5406a-1 {}] regionserver.HStore(327): Store=6e3b749fcbfdadfe59f1d1cf94f5406a/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:43,799 INFO [StoreOpener-386f4af1e738f142f3fbde37773d4d2c-1 {}] regionserver.HStore(327): Store=386f4af1e738f142f3fbde37773d4d2c/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:43,799 INFO [StoreOpener-386f4af1e738f142f3fbde37773d4d2c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 386f4af1e738f142f3fbde37773d4d2c 2024-11-15T21:54:43,800 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731707682333.46b5f83abe9585829b26884a1b26b43f. 2024-11-15T21:54:43,800 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731707682333.46b5f83abe9585829b26884a1b26b43f. 2024-11-15T21:54:43,800 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731707682333.0890aeec1f0ca0649e6483a2d56f36b4. 2024-11-15T21:54:43,801 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 0890aeec1f0ca0649e6483a2d56f36b4, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731707682333.0890aeec1f0ca0649e6483a2d56f36b4.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-15T21:54:43,801 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 0890aeec1f0ca0649e6483a2d56f36b4 2024-11-15T21:54:43,802 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731707682333.0890aeec1f0ca0649e6483a2d56f36b4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:43,802 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 0890aeec1f0ca0649e6483a2d56f36b4 2024-11-15T21:54:43,802 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 0890aeec1f0ca0649e6483a2d56f36b4 2024-11-15T21:54:43,802 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for 6e3b749fcbfdadfe59f1d1cf94f5406a 2024-11-15T21:54:43,804 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6e3b749fcbfdadfe59f1d1cf94f5406a 2024-11-15T21:54:43,804 INFO [StoreOpener-386f4af1e738f142f3fbde37773d4d2c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 386f4af1e738f142f3fbde37773d4d2c columnFamilyName colfamily21 2024-11-15T21:54:43,804 DEBUG [StoreOpener-386f4af1e738f142f3fbde37773d4d2c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:43,804 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6e3b749fcbfdadfe59f1d1cf94f5406a 2024-11-15T21:54:43,805 INFO [StoreOpener-386f4af1e738f142f3fbde37773d4d2c-1 {}] regionserver.HStore(327): Store=386f4af1e738f142f3fbde37773d4d2c/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:43,805 INFO [StoreOpener-386f4af1e738f142f3fbde37773d4d2c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 386f4af1e738f142f3fbde37773d4d2c 2024-11-15T21:54:43,805 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=26 2024-11-15T21:54:43,805 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 5ce283f02f0974e39c98759bb2bc9883, server=ae3311ea5ccb,40941,1731707679516 in 204 msec 2024-11-15T21:54:43,805 INFO [StoreOpener-0890aeec1f0ca0649e6483a2d56f36b4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 0890aeec1f0ca0649e6483a2d56f36b4 2024-11-15T21:54:43,808 INFO [StoreOpener-386f4af1e738f142f3fbde37773d4d2c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 386f4af1e738f142f3fbde37773d4d2c columnFamilyName colfamily31 2024-11-15T21:54:43,808 INFO [StoreOpener-0890aeec1f0ca0649e6483a2d56f36b4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0890aeec1f0ca0649e6483a2d56f36b4 columnFamilyName colfamily11 2024-11-15T21:54:43,808 DEBUG [StoreOpener-386f4af1e738f142f3fbde37773d4d2c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:43,808 DEBUG [StoreOpener-0890aeec1f0ca0649e6483a2d56f36b4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:43,808 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for 6e3b749fcbfdadfe59f1d1cf94f5406a 2024-11-15T21:54:43,809 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for 6e3b749fcbfdadfe59f1d1cf94f5406a 2024-11-15T21:54:43,810 INFO [StoreOpener-0890aeec1f0ca0649e6483a2d56f36b4-1 {}] regionserver.HStore(327): Store=0890aeec1f0ca0649e6483a2d56f36b4/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:43,811 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5ce283f02f0974e39c98759bb2bc9883, ASSIGN in 491 msec 2024-11-15T21:54:43,811 INFO [StoreOpener-0890aeec1f0ca0649e6483a2d56f36b4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 0890aeec1f0ca0649e6483a2d56f36b4 2024-11-15T21:54:43,811 INFO [StoreOpener-386f4af1e738f142f3fbde37773d4d2c-1 {}] regionserver.HStore(327): Store=386f4af1e738f142f3fbde37773d4d2c/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:43,812 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for 386f4af1e738f142f3fbde37773d4d2c 2024-11-15T21:54:43,812 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=46b5f83abe9585829b26884a1b26b43f, regionState=OPEN, openSeqNum=2, regionLocation=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:54:43,813 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/386f4af1e738f142f3fbde37773d4d2c 2024-11-15T21:54:43,814 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/386f4af1e738f142f3fbde37773d4d2c 2024-11-15T21:54:43,814 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-15T21:54:43,814 INFO [StoreOpener-0890aeec1f0ca0649e6483a2d56f36b4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0890aeec1f0ca0649e6483a2d56f36b4 columnFamilyName colfamily21 2024-11-15T21:54:43,815 DEBUG [StoreOpener-0890aeec1f0ca0649e6483a2d56f36b4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:43,816 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for 386f4af1e738f142f3fbde37773d4d2c 2024-11-15T21:54:43,816 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for 386f4af1e738f142f3fbde37773d4d2c 2024-11-15T21:54:43,816 INFO [StoreOpener-0890aeec1f0ca0649e6483a2d56f36b4-1 {}] regionserver.HStore(327): Store=0890aeec1f0ca0649e6483a2d56f36b4/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:43,817 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-15T21:54:43,817 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for 6e3b749fcbfdadfe59f1d1cf94f5406a 2024-11-15T21:54:43,817 INFO [StoreOpener-0890aeec1f0ca0649e6483a2d56f36b4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 0890aeec1f0ca0649e6483a2d56f36b4 2024-11-15T21:54:43,819 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36531 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=ae3311ea5ccb,37455,1731707679725, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=46b5f83abe9585829b26884a1b26b43f. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T21:54:43,820 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 46b5f83abe9585829b26884a1b26b43f, server=ae3311ea5ccb,37455,1731707679725 because future has completed 2024-11-15T21:54:43,822 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for 386f4af1e738f142f3fbde37773d4d2c 2024-11-15T21:54:43,826 INFO [StoreOpener-0890aeec1f0ca0649e6483a2d56f36b4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0890aeec1f0ca0649e6483a2d56f36b4 columnFamilyName colfamily31 2024-11-15T21:54:43,826 DEBUG [StoreOpener-0890aeec1f0ca0649e6483a2d56f36b4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:43,828 INFO [StoreOpener-0890aeec1f0ca0649e6483a2d56f36b4-1 {}] regionserver.HStore(327): Store=0890aeec1f0ca0649e6483a2d56f36b4/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:43,828 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 0890aeec1f0ca0649e6483a2d56f36b4 2024-11-15T21:54:43,828 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6e3b749fcbfdadfe59f1d1cf94f5406a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T21:54:43,829 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened 6e3b749fcbfdadfe59f1d1cf94f5406a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60164636, jitterRate=-0.10347706079483032}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-15T21:54:43,829 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0890aeec1f0ca0649e6483a2d56f36b4 2024-11-15T21:54:43,829 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6e3b749fcbfdadfe59f1d1cf94f5406a 2024-11-15T21:54:43,830 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0890aeec1f0ca0649e6483a2d56f36b4 2024-11-15T21:54:43,830 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for 6e3b749fcbfdadfe59f1d1cf94f5406a: Running coprocessor pre-open hook at 1731707683775Writing region info on filesystem at 1731707683775Initializing all the Stores at 1731707683778 (+3 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707683778Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707683781 (+3 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707683781Cleaning up temporary data from old regions at 1731707683809 (+28 ms)Running coprocessor post-open hooks at 1731707683829 (+20 ms)Region opened successfully at 1731707683829 2024-11-15T21:54:43,832 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 0890aeec1f0ca0649e6483a2d56f36b4 2024-11-15T21:54:43,832 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 0890aeec1f0ca0649e6483a2d56f36b4 2024-11-15T21:54:43,832 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731707682333.6e3b749fcbfdadfe59f1d1cf94f5406a., pid=52, masterSystemTime=1731707683767 2024-11-15T21:54:43,833 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-15T21:54:43,833 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/386f4af1e738f142f3fbde37773d4d2c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T21:54:43,834 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened 386f4af1e738f142f3fbde37773d4d2c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61298432, jitterRate=-0.08658218383789062}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-15T21:54:43,835 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 386f4af1e738f142f3fbde37773d4d2c 2024-11-15T21:54:43,835 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for 386f4af1e738f142f3fbde37773d4d2c: Running coprocessor pre-open hook at 1731707683779Writing region info on filesystem at 1731707683779Initializing all the Stores at 1731707683781 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707683781Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707683784 (+3 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707683784Cleaning up temporary data from old regions at 1731707683816 (+32 ms)Running coprocessor post-open hooks at 1731707683835 (+19 ms)Region opened successfully at 1731707683835 2024-11-15T21:54:43,837 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 0890aeec1f0ca0649e6483a2d56f36b4 2024-11-15T21:54:43,837 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731707682333.6e3b749fcbfdadfe59f1d1cf94f5406a. 2024-11-15T21:54:43,837 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731707682333.386f4af1e738f142f3fbde37773d4d2c., pid=44, masterSystemTime=1731707683657 2024-11-15T21:54:43,837 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731707682333.6e3b749fcbfdadfe59f1d1cf94f5406a. 2024-11-15T21:54:43,839 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=28 2024-11-15T21:54:43,839 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure 46b5f83abe9585829b26884a1b26b43f, server=ae3311ea5ccb,37455,1731707679725 in 266 msec 2024-11-15T21:54:43,839 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731707682333.d6063c53955b74f03cc36bd578a6bd41. 2024-11-15T21:54:43,839 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => d6063c53955b74f03cc36bd578a6bd41, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731707682333.d6063c53955b74f03cc36bd578a6bd41.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-15T21:54:43,840 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true d6063c53955b74f03cc36bd578a6bd41 2024-11-15T21:54:43,840 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731707682333.d6063c53955b74f03cc36bd578a6bd41.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:43,840 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for d6063c53955b74f03cc36bd578a6bd41 2024-11-15T21:54:43,840 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for d6063c53955b74f03cc36bd578a6bd41 2024-11-15T21:54:43,841 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731707682333.386f4af1e738f142f3fbde37773d4d2c. 2024-11-15T21:54:43,841 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731707682333.386f4af1e738f142f3fbde37773d4d2c. 2024-11-15T21:54:43,841 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731707682333.a3f7be24d811f4d6177c28e08f25af2d. 2024-11-15T21:54:43,841 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => a3f7be24d811f4d6177c28e08f25af2d, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731707682333.a3f7be24d811f4d6177c28e08f25af2d.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-15T21:54:43,842 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true a3f7be24d811f4d6177c28e08f25af2d 2024-11-15T21:54:43,842 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731707682333.a3f7be24d811f4d6177c28e08f25af2d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:43,842 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for a3f7be24d811f4d6177c28e08f25af2d 2024-11-15T21:54:43,842 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for a3f7be24d811f4d6177c28e08f25af2d 2024-11-15T21:54:43,846 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0890aeec1f0ca0649e6483a2d56f36b4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T21:54:43,847 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=386f4af1e738f142f3fbde37773d4d2c, regionState=OPEN, openSeqNum=2, regionLocation=ae3311ea5ccb,40941,1731707679516 2024-11-15T21:54:43,850 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 0890aeec1f0ca0649e6483a2d56f36b4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74929207, jitterRate=0.11653219163417816}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-15T21:54:43,851 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0890aeec1f0ca0649e6483a2d56f36b4 2024-11-15T21:54:43,851 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 0890aeec1f0ca0649e6483a2d56f36b4: Running coprocessor pre-open hook at 1731707683802Writing region info on filesystem at 1731707683802Initializing all the Stores at 1731707683804 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707683804Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707683805 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707683805Cleaning up temporary data from old regions at 1731707683832 (+27 ms)Running coprocessor post-open hooks at 1731707683851 (+19 ms)Region opened successfully at 1731707683851 2024-11-15T21:54:43,852 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 386f4af1e738f142f3fbde37773d4d2c, server=ae3311ea5ccb,40941,1731707679516 because future has completed 2024-11-15T21:54:43,853 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731707682333.0890aeec1f0ca0649e6483a2d56f36b4., pid=35, masterSystemTime=1731707683686 2024-11-15T21:54:43,855 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=46b5f83abe9585829b26884a1b26b43f, ASSIGN in 537 msec 2024-11-15T21:54:43,858 INFO [StoreOpener-a3f7be24d811f4d6177c28e08f25af2d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region a3f7be24d811f4d6177c28e08f25af2d 2024-11-15T21:54:43,858 INFO [StoreOpener-d6063c53955b74f03cc36bd578a6bd41-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region d6063c53955b74f03cc36bd578a6bd41 2024-11-15T21:54:43,858 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731707682333.0890aeec1f0ca0649e6483a2d56f36b4. 2024-11-15T21:54:43,859 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731707682333.0890aeec1f0ca0649e6483a2d56f36b4. 2024-11-15T21:54:43,859 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731707682333.2c83200b2c422973a68afce1173aae30. 2024-11-15T21:54:43,860 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => 2c83200b2c422973a68afce1173aae30, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731707682333.2c83200b2c422973a68afce1173aae30.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-15T21:54:43,860 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2c83200b2c422973a68afce1173aae30 2024-11-15T21:54:43,861 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731707682333.2c83200b2c422973a68afce1173aae30.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:43,861 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for 2c83200b2c422973a68afce1173aae30 2024-11-15T21:54:43,861 INFO [StoreOpener-a3f7be24d811f4d6177c28e08f25af2d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a3f7be24d811f4d6177c28e08f25af2d columnFamilyName colfamily11 2024-11-15T21:54:43,861 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for 2c83200b2c422973a68afce1173aae30 2024-11-15T21:54:43,861 DEBUG [StoreOpener-a3f7be24d811f4d6177c28e08f25af2d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:43,862 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=23 2024-11-15T21:54:43,863 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 386f4af1e738f142f3fbde37773d4d2c, server=ae3311ea5ccb,40941,1731707679516 in 270 msec 2024-11-15T21:54:43,863 INFO [StoreOpener-a3f7be24d811f4d6177c28e08f25af2d-1 {}] regionserver.HStore(327): Store=a3f7be24d811f4d6177c28e08f25af2d/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:43,863 INFO [StoreOpener-a3f7be24d811f4d6177c28e08f25af2d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region a3f7be24d811f4d6177c28e08f25af2d 2024-11-15T21:54:43,864 INFO [StoreOpener-d6063c53955b74f03cc36bd578a6bd41-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d6063c53955b74f03cc36bd578a6bd41 columnFamilyName colfamily11 2024-11-15T21:54:43,864 DEBUG [StoreOpener-d6063c53955b74f03cc36bd578a6bd41-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:43,865 INFO [StoreOpener-d6063c53955b74f03cc36bd578a6bd41-1 {}] regionserver.HStore(327): Store=d6063c53955b74f03cc36bd578a6bd41/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:43,865 INFO [StoreOpener-d6063c53955b74f03cc36bd578a6bd41-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region d6063c53955b74f03cc36bd578a6bd41 2024-11-15T21:54:43,866 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=386f4af1e738f142f3fbde37773d4d2c, ASSIGN in 549 msec 2024-11-15T21:54:43,868 INFO [StoreOpener-a3f7be24d811f4d6177c28e08f25af2d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a3f7be24d811f4d6177c28e08f25af2d columnFamilyName colfamily21 2024-11-15T21:54:43,868 INFO [StoreOpener-d6063c53955b74f03cc36bd578a6bd41-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d6063c53955b74f03cc36bd578a6bd41 columnFamilyName colfamily21 2024-11-15T21:54:43,868 DEBUG [StoreOpener-a3f7be24d811f4d6177c28e08f25af2d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:43,868 DEBUG [StoreOpener-d6063c53955b74f03cc36bd578a6bd41-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:43,869 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=0890aeec1f0ca0649e6483a2d56f36b4, regionState=OPEN, openSeqNum=2, regionLocation=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:54:43,871 INFO [StoreOpener-a3f7be24d811f4d6177c28e08f25af2d-1 {}] regionserver.HStore(327): Store=a3f7be24d811f4d6177c28e08f25af2d/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:43,872 INFO [StoreOpener-d6063c53955b74f03cc36bd578a6bd41-1 {}] regionserver.HStore(327): Store=d6063c53955b74f03cc36bd578a6bd41/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:43,872 INFO [StoreOpener-a3f7be24d811f4d6177c28e08f25af2d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region a3f7be24d811f4d6177c28e08f25af2d 2024-11-15T21:54:43,872 INFO [StoreOpener-d6063c53955b74f03cc36bd578a6bd41-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region d6063c53955b74f03cc36bd578a6bd41 2024-11-15T21:54:43,873 INFO [StoreOpener-2c83200b2c422973a68afce1173aae30-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 2c83200b2c422973a68afce1173aae30 2024-11-15T21:54:43,874 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=6e3b749fcbfdadfe59f1d1cf94f5406a, regionState=OPEN, openSeqNum=2, regionLocation=ae3311ea5ccb,32871,1731707679650 2024-11-15T21:54:43,874 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0890aeec1f0ca0649e6483a2d56f36b4, server=ae3311ea5ccb,37455,1731707679725 because future has completed 2024-11-15T21:54:43,875 INFO [StoreOpener-a3f7be24d811f4d6177c28e08f25af2d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a3f7be24d811f4d6177c28e08f25af2d columnFamilyName colfamily31 2024-11-15T21:54:43,875 DEBUG [StoreOpener-a3f7be24d811f4d6177c28e08f25af2d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:43,875 INFO [StoreOpener-d6063c53955b74f03cc36bd578a6bd41-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d6063c53955b74f03cc36bd578a6bd41 columnFamilyName colfamily31 2024-11-15T21:54:43,875 DEBUG [StoreOpener-d6063c53955b74f03cc36bd578a6bd41-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:43,877 INFO [StoreOpener-d6063c53955b74f03cc36bd578a6bd41-1 {}] regionserver.HStore(327): Store=d6063c53955b74f03cc36bd578a6bd41/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:43,877 INFO [StoreOpener-a3f7be24d811f4d6177c28e08f25af2d-1 {}] regionserver.HStore(327): Store=a3f7be24d811f4d6177c28e08f25af2d/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:43,877 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for d6063c53955b74f03cc36bd578a6bd41 2024-11-15T21:54:43,877 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for a3f7be24d811f4d6177c28e08f25af2d 2024-11-15T21:54:43,878 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6e3b749fcbfdadfe59f1d1cf94f5406a, server=ae3311ea5ccb,32871,1731707679650 because future has completed 2024-11-15T21:54:43,879 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d6063c53955b74f03cc36bd578a6bd41 2024-11-15T21:54:43,879 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a3f7be24d811f4d6177c28e08f25af2d 2024-11-15T21:54:43,880 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a3f7be24d811f4d6177c28e08f25af2d 2024-11-15T21:54:43,880 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d6063c53955b74f03cc36bd578a6bd41 2024-11-15T21:54:43,880 INFO [StoreOpener-2c83200b2c422973a68afce1173aae30-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2c83200b2c422973a68afce1173aae30 columnFamilyName colfamily11 2024-11-15T21:54:43,880 DEBUG [StoreOpener-2c83200b2c422973a68afce1173aae30-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:43,881 INFO [StoreOpener-2c83200b2c422973a68afce1173aae30-1 {}] regionserver.HStore(327): Store=2c83200b2c422973a68afce1173aae30/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:43,881 INFO [StoreOpener-2c83200b2c422973a68afce1173aae30-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 2c83200b2c422973a68afce1173aae30 2024-11-15T21:54:43,882 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for a3f7be24d811f4d6177c28e08f25af2d 2024-11-15T21:54:43,882 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for a3f7be24d811f4d6177c28e08f25af2d 2024-11-15T21:54:43,883 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-15T21:54:43,883 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for d6063c53955b74f03cc36bd578a6bd41 2024-11-15T21:54:43,883 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for d6063c53955b74f03cc36bd578a6bd41 2024-11-15T21:54:43,884 INFO [StoreOpener-2c83200b2c422973a68afce1173aae30-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2c83200b2c422973a68afce1173aae30 columnFamilyName colfamily21 2024-11-15T21:54:43,885 DEBUG [StoreOpener-2c83200b2c422973a68afce1173aae30-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:43,886 INFO [StoreOpener-2c83200b2c422973a68afce1173aae30-1 {}] regionserver.HStore(327): Store=2c83200b2c422973a68afce1173aae30/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:43,886 INFO [StoreOpener-2c83200b2c422973a68afce1173aae30-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 2c83200b2c422973a68afce1173aae30 2024-11-15T21:54:43,886 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for a3f7be24d811f4d6177c28e08f25af2d 2024-11-15T21:54:43,886 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-15T21:54:43,890 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=12 2024-11-15T21:54:43,890 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 0890aeec1f0ca0649e6483a2d56f36b4, server=ae3311ea5ccb,37455,1731707679725 in 342 msec 2024-11-15T21:54:43,890 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for d6063c53955b74f03cc36bd578a6bd41 2024-11-15T21:54:43,890 INFO [StoreOpener-2c83200b2c422973a68afce1173aae30-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2c83200b2c422973a68afce1173aae30 columnFamilyName colfamily31 2024-11-15T21:54:43,890 DEBUG [StoreOpener-2c83200b2c422973a68afce1173aae30-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:43,891 INFO [StoreOpener-2c83200b2c422973a68afce1173aae30-1 {}] regionserver.HStore(327): Store=2c83200b2c422973a68afce1173aae30/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:43,892 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for 2c83200b2c422973a68afce1173aae30 2024-11-15T21:54:43,892 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a3f7be24d811f4d6177c28e08f25af2d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T21:54:43,893 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=0890aeec1f0ca0649e6483a2d56f36b4, ASSIGN in 576 msec 2024-11-15T21:54:43,893 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2c83200b2c422973a68afce1173aae30 2024-11-15T21:54:43,894 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2c83200b2c422973a68afce1173aae30 2024-11-15T21:54:43,894 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened a3f7be24d811f4d6177c28e08f25af2d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66999534, jitterRate=-0.0016291439533233643}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-15T21:54:43,894 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a3f7be24d811f4d6177c28e08f25af2d 2024-11-15T21:54:43,894 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for a3f7be24d811f4d6177c28e08f25af2d: Running coprocessor pre-open hook at 1731707683842Writing region info on filesystem at 1731707683842Initializing all the Stores at 1731707683843 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707683843Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707683857 (+14 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707683857Cleaning up temporary data from old regions at 1731707683882 (+25 ms)Running coprocessor post-open hooks at 1731707683894 (+12 ms)Region opened successfully at 1731707683894 2024-11-15T21:54:43,896 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for 2c83200b2c422973a68afce1173aae30 2024-11-15T21:54:43,896 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for 2c83200b2c422973a68afce1173aae30 2024-11-15T21:54:43,897 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-15T21:54:43,897 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731707682333.a3f7be24d811f4d6177c28e08f25af2d., pid=43, masterSystemTime=1731707683657 2024-11-15T21:54:43,899 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for 2c83200b2c422973a68afce1173aae30 2024-11-15T21:54:43,900 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=17 2024-11-15T21:54:43,900 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 6e3b749fcbfdadfe59f1d1cf94f5406a, server=ae3311ea5ccb,32871,1731707679650 in 251 msec 2024-11-15T21:54:43,903 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=6e3b749fcbfdadfe59f1d1cf94f5406a, ASSIGN in 586 msec 2024-11-15T21:54:43,906 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d6063c53955b74f03cc36bd578a6bd41/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T21:54:43,907 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened d6063c53955b74f03cc36bd578a6bd41; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60595515, jitterRate=-0.09705646336078644}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-15T21:54:43,907 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d6063c53955b74f03cc36bd578a6bd41 2024-11-15T21:54:43,908 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for d6063c53955b74f03cc36bd578a6bd41: Running coprocessor pre-open hook at 1731707683840Writing region info on filesystem at 1731707683840Initializing all the Stores at 1731707683842 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707683842Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707683857 (+15 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707683857Cleaning up temporary data from old regions at 1731707683884 (+27 ms)Running coprocessor post-open hooks at 1731707683907 (+23 ms)Region opened successfully at 1731707683907 2024-11-15T21:54:43,909 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731707682333.a3f7be24d811f4d6177c28e08f25af2d. 2024-11-15T21:54:43,909 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731707682333.a3f7be24d811f4d6177c28e08f25af2d. 2024-11-15T21:54:43,909 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731707682333.e76ae188febdca91d319999f619d9e11. 2024-11-15T21:54:43,909 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => e76ae188febdca91d319999f619d9e11, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731707682333.e76ae188febdca91d319999f619d9e11.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-15T21:54:43,910 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true e76ae188febdca91d319999f619d9e11 2024-11-15T21:54:43,910 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731707682333.e76ae188febdca91d319999f619d9e11.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:43,910 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731707682333.d6063c53955b74f03cc36bd578a6bd41., pid=49, masterSystemTime=1731707683767 2024-11-15T21:54:43,910 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for e76ae188febdca91d319999f619d9e11 2024-11-15T21:54:43,910 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for e76ae188febdca91d319999f619d9e11 2024-11-15T21:54:43,911 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2c83200b2c422973a68afce1173aae30/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T21:54:43,912 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened 2c83200b2c422973a68afce1173aae30; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65166058, jitterRate=-0.02895006537437439}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-15T21:54:43,913 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2c83200b2c422973a68afce1173aae30 2024-11-15T21:54:43,913 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for 2c83200b2c422973a68afce1173aae30: Running coprocessor pre-open hook at 1731707683862Writing region info on filesystem at 1731707683862Initializing all the Stores at 1731707683865 (+3 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707683865Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707683873 (+8 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707683873Cleaning up temporary data from old regions at 1731707683896 (+23 ms)Running coprocessor post-open hooks at 1731707683913 (+17 ms)Region opened successfully at 1731707683913 2024-11-15T21:54:43,914 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=a3f7be24d811f4d6177c28e08f25af2d, regionState=OPEN, openSeqNum=2, regionLocation=ae3311ea5ccb,40941,1731707679516 2024-11-15T21:54:43,915 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731707682333.d6063c53955b74f03cc36bd578a6bd41. 2024-11-15T21:54:43,915 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731707682333.2c83200b2c422973a68afce1173aae30., pid=46, masterSystemTime=1731707683686 2024-11-15T21:54:43,915 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731707682333.d6063c53955b74f03cc36bd578a6bd41. 2024-11-15T21:54:43,915 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731707682333.f9ef867a298d175eca9d85304f2faa69. 2024-11-15T21:54:43,915 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => f9ef867a298d175eca9d85304f2faa69, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731707682333.f9ef867a298d175eca9d85304f2faa69.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-15T21:54:43,916 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true f9ef867a298d175eca9d85304f2faa69 2024-11-15T21:54:43,916 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731707682333.f9ef867a298d175eca9d85304f2faa69.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:43,916 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for f9ef867a298d175eca9d85304f2faa69 2024-11-15T21:54:43,916 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for f9ef867a298d175eca9d85304f2faa69 2024-11-15T21:54:43,918 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731707682333.2c83200b2c422973a68afce1173aae30. 2024-11-15T21:54:43,918 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure a3f7be24d811f4d6177c28e08f25af2d, server=ae3311ea5ccb,40941,1731707679516 because future has completed 2024-11-15T21:54:43,918 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731707682333.2c83200b2c422973a68afce1173aae30. 2024-11-15T21:54:43,918 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731707682333.cbb6b5b8341f7ff2ada4ac1931a1a588. 2024-11-15T21:54:43,919 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => cbb6b5b8341f7ff2ada4ac1931a1a588, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731707682333.cbb6b5b8341f7ff2ada4ac1931a1a588.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-15T21:54:43,919 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true cbb6b5b8341f7ff2ada4ac1931a1a588 2024-11-15T21:54:43,919 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731707682333.cbb6b5b8341f7ff2ada4ac1931a1a588.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:43,919 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for cbb6b5b8341f7ff2ada4ac1931a1a588 2024-11-15T21:54:43,919 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for cbb6b5b8341f7ff2ada4ac1931a1a588 2024-11-15T21:54:43,920 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=2c83200b2c422973a68afce1173aae30, regionState=OPEN, openSeqNum=2, regionLocation=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:54:43,924 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36531 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=ae3311ea5ccb,37455,1731707679725, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2c83200b2c422973a68afce1173aae30. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T21:54:43,925 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36531 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=ae3311ea5ccb,40941,1731707679516, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a3f7be24d811f4d6177c28e08f25af2d. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T21:54:43,926 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2c83200b2c422973a68afce1173aae30, server=ae3311ea5ccb,37455,1731707679725 because future has completed 2024-11-15T21:54:43,929 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=16 2024-11-15T21:54:43,929 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure a3f7be24d811f4d6177c28e08f25af2d, server=ae3311ea5ccb,40941,1731707679516 in 344 msec 2024-11-15T21:54:43,930 INFO [StoreOpener-e76ae188febdca91d319999f619d9e11-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region e76ae188febdca91d319999f619d9e11 2024-11-15T21:54:43,932 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a3f7be24d811f4d6177c28e08f25af2d, ASSIGN in 615 msec 2024-11-15T21:54:43,933 INFO [StoreOpener-e76ae188febdca91d319999f619d9e11-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e76ae188febdca91d319999f619d9e11 columnFamilyName colfamily11 2024-11-15T21:54:43,933 DEBUG [StoreOpener-e76ae188febdca91d319999f619d9e11-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:43,934 INFO [StoreOpener-cbb6b5b8341f7ff2ada4ac1931a1a588-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region cbb6b5b8341f7ff2ada4ac1931a1a588 2024-11-15T21:54:43,936 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=9 2024-11-15T21:54:43,936 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure 2c83200b2c422973a68afce1173aae30, server=ae3311ea5ccb,37455,1731707679725 in 329 msec 2024-11-15T21:54:43,938 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2c83200b2c422973a68afce1173aae30, ASSIGN in 622 msec 2024-11-15T21:54:43,939 INFO [StoreOpener-e76ae188febdca91d319999f619d9e11-1 {}] regionserver.HStore(327): Store=e76ae188febdca91d319999f619d9e11/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:43,939 INFO [StoreOpener-e76ae188febdca91d319999f619d9e11-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region e76ae188febdca91d319999f619d9e11 2024-11-15T21:54:43,940 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=d6063c53955b74f03cc36bd578a6bd41, regionState=OPEN, openSeqNum=2, regionLocation=ae3311ea5ccb,32871,1731707679650 2024-11-15T21:54:43,942 INFO [StoreOpener-e76ae188febdca91d319999f619d9e11-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e76ae188febdca91d319999f619d9e11 columnFamilyName colfamily21 2024-11-15T21:54:43,942 DEBUG [StoreOpener-e76ae188febdca91d319999f619d9e11-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:43,942 INFO [StoreOpener-cbb6b5b8341f7ff2ada4ac1931a1a588-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cbb6b5b8341f7ff2ada4ac1931a1a588 columnFamilyName colfamily11 2024-11-15T21:54:43,942 DEBUG [StoreOpener-cbb6b5b8341f7ff2ada4ac1931a1a588-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:43,943 INFO [StoreOpener-cbb6b5b8341f7ff2ada4ac1931a1a588-1 {}] regionserver.HStore(327): Store=cbb6b5b8341f7ff2ada4ac1931a1a588/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:43,943 INFO [StoreOpener-cbb6b5b8341f7ff2ada4ac1931a1a588-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region cbb6b5b8341f7ff2ada4ac1931a1a588 2024-11-15T21:54:43,943 INFO [StoreOpener-e76ae188febdca91d319999f619d9e11-1 {}] regionserver.HStore(327): Store=e76ae188febdca91d319999f619d9e11/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:43,944 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure d6063c53955b74f03cc36bd578a6bd41, server=ae3311ea5ccb,32871,1731707679650 because future has completed 2024-11-15T21:54:43,944 INFO [StoreOpener-e76ae188febdca91d319999f619d9e11-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region e76ae188febdca91d319999f619d9e11 2024-11-15T21:54:43,946 INFO [StoreOpener-e76ae188febdca91d319999f619d9e11-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e76ae188febdca91d319999f619d9e11 columnFamilyName colfamily31 2024-11-15T21:54:43,946 DEBUG [StoreOpener-e76ae188febdca91d319999f619d9e11-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:43,947 INFO [StoreOpener-cbb6b5b8341f7ff2ada4ac1931a1a588-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cbb6b5b8341f7ff2ada4ac1931a1a588 columnFamilyName colfamily21 2024-11-15T21:54:43,947 DEBUG [StoreOpener-cbb6b5b8341f7ff2ada4ac1931a1a588-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:43,947 INFO [StoreOpener-e76ae188febdca91d319999f619d9e11-1 {}] regionserver.HStore(327): Store=e76ae188febdca91d319999f619d9e11/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:43,948 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for e76ae188febdca91d319999f619d9e11 2024-11-15T21:54:43,948 INFO [StoreOpener-cbb6b5b8341f7ff2ada4ac1931a1a588-1 {}] regionserver.HStore(327): Store=cbb6b5b8341f7ff2ada4ac1931a1a588/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:43,948 INFO [StoreOpener-cbb6b5b8341f7ff2ada4ac1931a1a588-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region cbb6b5b8341f7ff2ada4ac1931a1a588 2024-11-15T21:54:43,949 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e76ae188febdca91d319999f619d9e11 2024-11-15T21:54:43,950 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e76ae188febdca91d319999f619d9e11 2024-11-15T21:54:43,950 INFO [StoreOpener-cbb6b5b8341f7ff2ada4ac1931a1a588-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cbb6b5b8341f7ff2ada4ac1931a1a588 columnFamilyName colfamily31 2024-11-15T21:54:43,951 DEBUG [StoreOpener-cbb6b5b8341f7ff2ada4ac1931a1a588-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:43,951 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=24 2024-11-15T21:54:43,951 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure d6063c53955b74f03cc36bd578a6bd41, server=ae3311ea5ccb,32871,1731707679650 in 323 msec 2024-11-15T21:54:43,951 INFO [StoreOpener-cbb6b5b8341f7ff2ada4ac1931a1a588-1 {}] regionserver.HStore(327): Store=cbb6b5b8341f7ff2ada4ac1931a1a588/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:43,952 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for cbb6b5b8341f7ff2ada4ac1931a1a588 2024-11-15T21:54:43,952 INFO [StoreOpener-f9ef867a298d175eca9d85304f2faa69-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region f9ef867a298d175eca9d85304f2faa69 2024-11-15T21:54:43,953 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for e76ae188febdca91d319999f619d9e11 2024-11-15T21:54:43,953 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for e76ae188febdca91d319999f619d9e11 2024-11-15T21:54:43,953 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/cbb6b5b8341f7ff2ada4ac1931a1a588 2024-11-15T21:54:43,954 INFO [StoreOpener-f9ef867a298d175eca9d85304f2faa69-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f9ef867a298d175eca9d85304f2faa69 columnFamilyName colfamily11 2024-11-15T21:54:43,954 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/cbb6b5b8341f7ff2ada4ac1931a1a588 2024-11-15T21:54:43,954 DEBUG [StoreOpener-f9ef867a298d175eca9d85304f2faa69-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:43,956 INFO [StoreOpener-f9ef867a298d175eca9d85304f2faa69-1 {}] regionserver.HStore(327): Store=f9ef867a298d175eca9d85304f2faa69/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:43,956 INFO [StoreOpener-f9ef867a298d175eca9d85304f2faa69-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region f9ef867a298d175eca9d85304f2faa69 2024-11-15T21:54:43,958 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-15T21:54:43,959 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d6063c53955b74f03cc36bd578a6bd41, ASSIGN in 637 msec 2024-11-15T21:54:43,959 INFO [StoreOpener-f9ef867a298d175eca9d85304f2faa69-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f9ef867a298d175eca9d85304f2faa69 columnFamilyName colfamily21 2024-11-15T21:54:43,959 DEBUG [StoreOpener-f9ef867a298d175eca9d85304f2faa69-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:43,960 INFO [StoreOpener-f9ef867a298d175eca9d85304f2faa69-1 {}] regionserver.HStore(327): Store=f9ef867a298d175eca9d85304f2faa69/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:43,960 INFO [StoreOpener-f9ef867a298d175eca9d85304f2faa69-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region f9ef867a298d175eca9d85304f2faa69 2024-11-15T21:54:43,962 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for cbb6b5b8341f7ff2ada4ac1931a1a588 2024-11-15T21:54:43,962 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for cbb6b5b8341f7ff2ada4ac1931a1a588 2024-11-15T21:54:43,963 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-15T21:54:43,963 INFO [StoreOpener-f9ef867a298d175eca9d85304f2faa69-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f9ef867a298d175eca9d85304f2faa69 columnFamilyName colfamily31 2024-11-15T21:54:43,963 DEBUG [StoreOpener-f9ef867a298d175eca9d85304f2faa69-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:43,964 INFO [StoreOpener-f9ef867a298d175eca9d85304f2faa69-1 {}] regionserver.HStore(327): Store=f9ef867a298d175eca9d85304f2faa69/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:43,964 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for f9ef867a298d175eca9d85304f2faa69 2024-11-15T21:54:43,965 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for cbb6b5b8341f7ff2ada4ac1931a1a588 2024-11-15T21:54:43,965 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f9ef867a298d175eca9d85304f2faa69 2024-11-15T21:54:43,965 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for e76ae188febdca91d319999f619d9e11 2024-11-15T21:54:43,966 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f9ef867a298d175eca9d85304f2faa69 2024-11-15T21:54:43,969 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for f9ef867a298d175eca9d85304f2faa69 2024-11-15T21:54:43,969 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for f9ef867a298d175eca9d85304f2faa69 2024-11-15T21:54:43,970 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-15T21:54:43,972 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for f9ef867a298d175eca9d85304f2faa69 2024-11-15T21:54:43,974 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/cbb6b5b8341f7ff2ada4ac1931a1a588/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T21:54:43,975 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened cbb6b5b8341f7ff2ada4ac1931a1a588; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65208934, jitterRate=-0.028311163187026978}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-15T21:54:43,975 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cbb6b5b8341f7ff2ada4ac1931a1a588 2024-11-15T21:54:43,975 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for cbb6b5b8341f7ff2ada4ac1931a1a588: Running coprocessor pre-open hook at 1731707683919Writing region info on filesystem at 1731707683919Initializing all the Stores at 1731707683922 (+3 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707683922Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707683933 (+11 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707683933Cleaning up temporary data from old regions at 1731707683962 (+29 ms)Running coprocessor post-open hooks at 1731707683975 (+13 ms)Region opened successfully at 1731707683975 2024-11-15T21:54:43,977 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e76ae188febdca91d319999f619d9e11/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T21:54:43,977 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731707682333.cbb6b5b8341f7ff2ada4ac1931a1a588., pid=36, masterSystemTime=1731707683686 2024-11-15T21:54:43,978 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened e76ae188febdca91d319999f619d9e11; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72008848, jitterRate=0.07301545143127441}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-15T21:54:43,978 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e76ae188febdca91d319999f619d9e11 2024-11-15T21:54:43,978 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for e76ae188febdca91d319999f619d9e11: Running coprocessor pre-open hook at 1731707683910Writing region info on filesystem at 1731707683910Initializing all the Stores at 1731707683912 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707683912Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707683929 (+17 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707683929Cleaning up temporary data from old regions at 1731707683953 (+24 ms)Running coprocessor post-open hooks at 1731707683978 (+25 ms)Region opened successfully at 1731707683978 2024-11-15T21:54:43,978 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f9ef867a298d175eca9d85304f2faa69/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T21:54:43,979 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened f9ef867a298d175eca9d85304f2faa69; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66915976, jitterRate=-0.0028742551803588867}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-15T21:54:43,979 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f9ef867a298d175eca9d85304f2faa69 2024-11-15T21:54:43,979 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for f9ef867a298d175eca9d85304f2faa69: Running coprocessor pre-open hook at 1731707683916Writing region info on filesystem at 1731707683916Initializing all the Stores at 1731707683918 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707683918Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707683941 (+23 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707683942 (+1 ms)Cleaning up temporary data from old regions at 1731707683969 (+27 ms)Running coprocessor post-open hooks at 1731707683979 (+10 ms)Region opened successfully at 1731707683979 2024-11-15T21:54:43,979 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731707682333.e76ae188febdca91d319999f619d9e11., pid=31, masterSystemTime=1731707683657 2024-11-15T21:54:43,981 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731707682333.f9ef867a298d175eca9d85304f2faa69., pid=54, masterSystemTime=1731707683767 2024-11-15T21:54:43,985 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=e76ae188febdca91d319999f619d9e11, regionState=OPEN, openSeqNum=2, regionLocation=ae3311ea5ccb,40941,1731707679516 2024-11-15T21:54:43,986 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731707682333.cbb6b5b8341f7ff2ada4ac1931a1a588. 2024-11-15T21:54:43,986 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731707682333.cbb6b5b8341f7ff2ada4ac1931a1a588. 2024-11-15T21:54:43,986 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731707682333.1031049c0b4149d1189db33e38ab56cd. 2024-11-15T21:54:43,986 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => 1031049c0b4149d1189db33e38ab56cd, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731707682333.1031049c0b4149d1189db33e38ab56cd.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-15T21:54:43,987 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 1031049c0b4149d1189db33e38ab56cd 2024-11-15T21:54:43,987 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731707682333.1031049c0b4149d1189db33e38ab56cd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:43,987 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for 1031049c0b4149d1189db33e38ab56cd 2024-11-15T21:54:43,987 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for 1031049c0b4149d1189db33e38ab56cd 2024-11-15T21:54:43,988 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731707682333.e76ae188febdca91d319999f619d9e11. 2024-11-15T21:54:43,988 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731707682333.e76ae188febdca91d319999f619d9e11. 2024-11-15T21:54:43,989 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731707682333.d28ee8a2ac06d2199c72ac7f884ba470. 2024-11-15T21:54:43,989 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure e76ae188febdca91d319999f619d9e11, server=ae3311ea5ccb,40941,1731707679516 because future has completed 2024-11-15T21:54:43,989 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => d28ee8a2ac06d2199c72ac7f884ba470, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731707682333.d28ee8a2ac06d2199c72ac7f884ba470.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-15T21:54:43,989 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731707682333.f9ef867a298d175eca9d85304f2faa69. 2024-11-15T21:54:43,989 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731707682333.f9ef867a298d175eca9d85304f2faa69. 2024-11-15T21:54:43,989 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731707682333.6694325f9c862a5a1ee02eabf7861cc3. 2024-11-15T21:54:43,989 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true d28ee8a2ac06d2199c72ac7f884ba470 2024-11-15T21:54:43,989 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => 6694325f9c862a5a1ee02eabf7861cc3, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731707682333.6694325f9c862a5a1ee02eabf7861cc3.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-15T21:54:43,989 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731707682333.d28ee8a2ac06d2199c72ac7f884ba470.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:43,990 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for d28ee8a2ac06d2199c72ac7f884ba470 2024-11-15T21:54:43,990 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 6694325f9c862a5a1ee02eabf7861cc3 2024-11-15T21:54:43,990 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for d28ee8a2ac06d2199c72ac7f884ba470 2024-11-15T21:54:43,990 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731707682333.6694325f9c862a5a1ee02eabf7861cc3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:43,990 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for 6694325f9c862a5a1ee02eabf7861cc3 2024-11-15T21:54:43,990 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for 6694325f9c862a5a1ee02eabf7861cc3 2024-11-15T21:54:43,993 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=f9ef867a298d175eca9d85304f2faa69, regionState=OPEN, openSeqNum=2, regionLocation=ae3311ea5ccb,32871,1731707679650 2024-11-15T21:54:43,997 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure f9ef867a298d175eca9d85304f2faa69, server=ae3311ea5ccb,32871,1731707679650 because future has completed 2024-11-15T21:54:44,003 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-15T21:54:44,003 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure e76ae188febdca91d319999f619d9e11, server=ae3311ea5ccb,40941,1731707679516 in 497 msec 2024-11-15T21:54:44,006 INFO [StoreOpener-d28ee8a2ac06d2199c72ac7f884ba470-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region d28ee8a2ac06d2199c72ac7f884ba470 2024-11-15T21:54:44,010 INFO [StoreOpener-1031049c0b4149d1189db33e38ab56cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 1031049c0b4149d1189db33e38ab56cd 2024-11-15T21:54:44,010 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e76ae188febdca91d319999f619d9e11, ASSIGN in 689 msec 2024-11-15T21:54:44,012 INFO [StoreOpener-d28ee8a2ac06d2199c72ac7f884ba470-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d28ee8a2ac06d2199c72ac7f884ba470 columnFamilyName colfamily11 2024-11-15T21:54:44,013 DEBUG [StoreOpener-d28ee8a2ac06d2199c72ac7f884ba470-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:44,013 INFO [StoreOpener-1031049c0b4149d1189db33e38ab56cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1031049c0b4149d1189db33e38ab56cd columnFamilyName colfamily11 2024-11-15T21:54:44,013 DEBUG [StoreOpener-1031049c0b4149d1189db33e38ab56cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:44,014 INFO [StoreOpener-d28ee8a2ac06d2199c72ac7f884ba470-1 {}] regionserver.HStore(327): Store=d28ee8a2ac06d2199c72ac7f884ba470/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:44,014 INFO [StoreOpener-1031049c0b4149d1189db33e38ab56cd-1 {}] regionserver.HStore(327): Store=1031049c0b4149d1189db33e38ab56cd/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:44,014 INFO [StoreOpener-d28ee8a2ac06d2199c72ac7f884ba470-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region d28ee8a2ac06d2199c72ac7f884ba470 2024-11-15T21:54:44,014 INFO [StoreOpener-1031049c0b4149d1189db33e38ab56cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 1031049c0b4149d1189db33e38ab56cd 2024-11-15T21:54:44,016 INFO [StoreOpener-d28ee8a2ac06d2199c72ac7f884ba470-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d28ee8a2ac06d2199c72ac7f884ba470 columnFamilyName colfamily21 2024-11-15T21:54:44,016 DEBUG [StoreOpener-d28ee8a2ac06d2199c72ac7f884ba470-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:44,016 INFO [StoreOpener-1031049c0b4149d1189db33e38ab56cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1031049c0b4149d1189db33e38ab56cd columnFamilyName colfamily21 2024-11-15T21:54:44,016 DEBUG [StoreOpener-1031049c0b4149d1189db33e38ab56cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:44,017 INFO [StoreOpener-d28ee8a2ac06d2199c72ac7f884ba470-1 {}] regionserver.HStore(327): Store=d28ee8a2ac06d2199c72ac7f884ba470/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:44,017 INFO [StoreOpener-1031049c0b4149d1189db33e38ab56cd-1 {}] regionserver.HStore(327): Store=1031049c0b4149d1189db33e38ab56cd/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:44,017 INFO [StoreOpener-d28ee8a2ac06d2199c72ac7f884ba470-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region d28ee8a2ac06d2199c72ac7f884ba470 2024-11-15T21:54:44,017 INFO [StoreOpener-1031049c0b4149d1189db33e38ab56cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 1031049c0b4149d1189db33e38ab56cd 2024-11-15T21:54:44,017 INFO [StoreOpener-6694325f9c862a5a1ee02eabf7861cc3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 6694325f9c862a5a1ee02eabf7861cc3 2024-11-15T21:54:44,020 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=11 2024-11-15T21:54:44,020 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure f9ef867a298d175eca9d85304f2faa69, server=ae3311ea5ccb,32871,1731707679650 in 358 msec 2024-11-15T21:54:44,020 INFO [StoreOpener-d28ee8a2ac06d2199c72ac7f884ba470-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d28ee8a2ac06d2199c72ac7f884ba470 columnFamilyName colfamily31 2024-11-15T21:54:44,021 INFO [StoreOpener-1031049c0b4149d1189db33e38ab56cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1031049c0b4149d1189db33e38ab56cd columnFamilyName colfamily31 2024-11-15T21:54:44,021 INFO [StoreOpener-6694325f9c862a5a1ee02eabf7861cc3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6694325f9c862a5a1ee02eabf7861cc3 columnFamilyName colfamily11 2024-11-15T21:54:44,021 DEBUG [StoreOpener-1031049c0b4149d1189db33e38ab56cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:44,021 DEBUG [StoreOpener-d28ee8a2ac06d2199c72ac7f884ba470-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:44,021 DEBUG [StoreOpener-6694325f9c862a5a1ee02eabf7861cc3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:44,022 INFO [StoreOpener-1031049c0b4149d1189db33e38ab56cd-1 {}] regionserver.HStore(327): Store=1031049c0b4149d1189db33e38ab56cd/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:44,022 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for 1031049c0b4149d1189db33e38ab56cd 2024-11-15T21:54:44,022 INFO [StoreOpener-d28ee8a2ac06d2199c72ac7f884ba470-1 {}] regionserver.HStore(327): Store=d28ee8a2ac06d2199c72ac7f884ba470/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:44,022 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for d28ee8a2ac06d2199c72ac7f884ba470 2024-11-15T21:54:44,023 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1031049c0b4149d1189db33e38ab56cd 2024-11-15T21:54:44,024 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d28ee8a2ac06d2199c72ac7f884ba470 2024-11-15T21:54:44,024 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1031049c0b4149d1189db33e38ab56cd 2024-11-15T21:54:44,024 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=f9ef867a298d175eca9d85304f2faa69, ASSIGN in 706 msec 2024-11-15T21:54:44,024 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d28ee8a2ac06d2199c72ac7f884ba470 2024-11-15T21:54:44,026 INFO [StoreOpener-6694325f9c862a5a1ee02eabf7861cc3-1 {}] regionserver.HStore(327): Store=6694325f9c862a5a1ee02eabf7861cc3/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:44,027 INFO [StoreOpener-6694325f9c862a5a1ee02eabf7861cc3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 6694325f9c862a5a1ee02eabf7861cc3 2024-11-15T21:54:44,027 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=cbb6b5b8341f7ff2ada4ac1931a1a588, regionState=OPEN, openSeqNum=2, regionLocation=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:54:44,028 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for 1031049c0b4149d1189db33e38ab56cd 2024-11-15T21:54:44,028 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for 1031049c0b4149d1189db33e38ab56cd 2024-11-15T21:54:44,028 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for d28ee8a2ac06d2199c72ac7f884ba470 2024-11-15T21:54:44,028 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for d28ee8a2ac06d2199c72ac7f884ba470 2024-11-15T21:54:44,029 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-15T21:54:44,029 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36531 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=ae3311ea5ccb,37455,1731707679725, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=cbb6b5b8341f7ff2ada4ac1931a1a588. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T21:54:44,030 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-15T21:54:44,031 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure cbb6b5b8341f7ff2ada4ac1931a1a588, server=ae3311ea5ccb,37455,1731707679725 because future has completed 2024-11-15T21:54:44,031 INFO [StoreOpener-6694325f9c862a5a1ee02eabf7861cc3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6694325f9c862a5a1ee02eabf7861cc3 columnFamilyName colfamily21 2024-11-15T21:54:44,031 DEBUG [StoreOpener-6694325f9c862a5a1ee02eabf7861cc3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:44,031 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for 1031049c0b4149d1189db33e38ab56cd 2024-11-15T21:54:44,032 INFO [StoreOpener-6694325f9c862a5a1ee02eabf7861cc3-1 {}] regionserver.HStore(327): Store=6694325f9c862a5a1ee02eabf7861cc3/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:44,032 INFO [StoreOpener-6694325f9c862a5a1ee02eabf7861cc3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 6694325f9c862a5a1ee02eabf7861cc3 2024-11-15T21:54:44,033 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for d28ee8a2ac06d2199c72ac7f884ba470 2024-11-15T21:54:44,037 INFO [StoreOpener-6694325f9c862a5a1ee02eabf7861cc3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6694325f9c862a5a1ee02eabf7861cc3 columnFamilyName colfamily31 2024-11-15T21:54:44,037 DEBUG [StoreOpener-6694325f9c862a5a1ee02eabf7861cc3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:44,038 INFO [StoreOpener-6694325f9c862a5a1ee02eabf7861cc3-1 {}] regionserver.HStore(327): Store=6694325f9c862a5a1ee02eabf7861cc3/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:44,038 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for 6694325f9c862a5a1ee02eabf7861cc3 2024-11-15T21:54:44,039 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6694325f9c862a5a1ee02eabf7861cc3 2024-11-15T21:54:44,040 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=15 2024-11-15T21:54:44,040 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure cbb6b5b8341f7ff2ada4ac1931a1a588, server=ae3311ea5ccb,37455,1731707679725 in 490 msec 2024-11-15T21:54:44,040 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6694325f9c862a5a1ee02eabf7861cc3 2024-11-15T21:54:44,041 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1031049c0b4149d1189db33e38ab56cd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T21:54:44,041 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened 1031049c0b4149d1189db33e38ab56cd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65511596, jitterRate=-0.023801147937774658}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-15T21:54:44,042 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for 6694325f9c862a5a1ee02eabf7861cc3 2024-11-15T21:54:44,042 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1031049c0b4149d1189db33e38ab56cd 2024-11-15T21:54:44,042 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for 6694325f9c862a5a1ee02eabf7861cc3 2024-11-15T21:54:44,042 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d28ee8a2ac06d2199c72ac7f884ba470/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T21:54:44,042 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for 1031049c0b4149d1189db33e38ab56cd: Running coprocessor pre-open hook at 1731707683988Writing region info on filesystem at 1731707683988Initializing all the Stores at 1731707683990 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707683990Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707684005 (+15 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707684005Cleaning up temporary data from old regions at 1731707684028 (+23 ms)Running coprocessor post-open hooks at 1731707684042 (+14 ms)Region opened successfully at 1731707684042 2024-11-15T21:54:44,043 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-15T21:54:44,043 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened d28ee8a2ac06d2199c72ac7f884ba470; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61174233, jitterRate=-0.08843289315700531}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-15T21:54:44,043 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d28ee8a2ac06d2199c72ac7f884ba470 2024-11-15T21:54:44,043 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for d28ee8a2ac06d2199c72ac7f884ba470: Running coprocessor pre-open hook at 1731707683990Writing region info on filesystem at 1731707683990Initializing all the Stores at 1731707683992 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707683992Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707684005 (+13 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707684006 (+1 ms)Cleaning up temporary data from old regions at 1731707684028 (+22 ms)Running coprocessor post-open hooks at 1731707684043 (+15 ms)Region opened successfully at 1731707684043 2024-11-15T21:54:44,044 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731707682333.1031049c0b4149d1189db33e38ab56cd., pid=40, masterSystemTime=1731707683686 2024-11-15T21:54:44,045 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731707682333.d28ee8a2ac06d2199c72ac7f884ba470., pid=42, masterSystemTime=1731707683657 2024-11-15T21:54:44,046 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for 6694325f9c862a5a1ee02eabf7861cc3 2024-11-15T21:54:44,046 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=cbb6b5b8341f7ff2ada4ac1931a1a588, ASSIGN in 726 msec 2024-11-15T21:54:44,048 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731707682333.1031049c0b4149d1189db33e38ab56cd. 2024-11-15T21:54:44,048 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731707682333.1031049c0b4149d1189db33e38ab56cd. 2024-11-15T21:54:44,049 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731707682333.51af260ba7a9b40a7f788a5c93a416b6. 2024-11-15T21:54:44,049 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 51af260ba7a9b40a7f788a5c93a416b6, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731707682333.51af260ba7a9b40a7f788a5c93a416b6.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-15T21:54:44,049 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 51af260ba7a9b40a7f788a5c93a416b6 2024-11-15T21:54:44,049 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731707682333.51af260ba7a9b40a7f788a5c93a416b6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:44,049 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 51af260ba7a9b40a7f788a5c93a416b6 2024-11-15T21:54:44,050 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 51af260ba7a9b40a7f788a5c93a416b6 2024-11-15T21:54:44,050 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=1031049c0b4149d1189db33e38ab56cd, regionState=OPEN, openSeqNum=2, regionLocation=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:54:44,052 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731707682333.d28ee8a2ac06d2199c72ac7f884ba470. 2024-11-15T21:54:44,052 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731707682333.d28ee8a2ac06d2199c72ac7f884ba470. 2024-11-15T21:54:44,052 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731707682333.0ab2d104e772f41ee9c56851adad43f0. 2024-11-15T21:54:44,052 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 0ab2d104e772f41ee9c56851adad43f0, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731707682333.0ab2d104e772f41ee9c56851adad43f0.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-15T21:54:44,053 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 0ab2d104e772f41ee9c56851adad43f0 2024-11-15T21:54:44,053 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731707682333.0ab2d104e772f41ee9c56851adad43f0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:44,053 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 0ab2d104e772f41ee9c56851adad43f0 2024-11-15T21:54:44,053 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 0ab2d104e772f41ee9c56851adad43f0 2024-11-15T21:54:44,054 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=d28ee8a2ac06d2199c72ac7f884ba470, regionState=OPEN, openSeqNum=2, regionLocation=ae3311ea5ccb,40941,1731707679516 2024-11-15T21:54:44,059 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1031049c0b4149d1189db33e38ab56cd, server=ae3311ea5ccb,37455,1731707679725 because future has completed 2024-11-15T21:54:44,060 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure d28ee8a2ac06d2199c72ac7f884ba470, server=ae3311ea5ccb,40941,1731707679516 because future has completed 2024-11-15T21:54:44,063 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6694325f9c862a5a1ee02eabf7861cc3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T21:54:44,063 INFO [StoreOpener-0ab2d104e772f41ee9c56851adad43f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 0ab2d104e772f41ee9c56851adad43f0 2024-11-15T21:54:44,064 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened 6694325f9c862a5a1ee02eabf7861cc3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68790571, jitterRate=0.02505938708782196}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-15T21:54:44,064 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6694325f9c862a5a1ee02eabf7861cc3 2024-11-15T21:54:44,064 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for 6694325f9c862a5a1ee02eabf7861cc3: Running coprocessor pre-open hook at 1731707683990Writing region info on filesystem at 1731707683990Initializing all the Stores at 1731707683992 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707683992Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707684008 (+16 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707684008Cleaning up temporary data from old regions at 1731707684042 (+34 ms)Running coprocessor post-open hooks at 1731707684064 (+22 ms)Region opened successfully at 1731707684064 2024-11-15T21:54:44,065 INFO [StoreOpener-51af260ba7a9b40a7f788a5c93a416b6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 51af260ba7a9b40a7f788a5c93a416b6 2024-11-15T21:54:44,066 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731707682333.6694325f9c862a5a1ee02eabf7861cc3., pid=50, masterSystemTime=1731707683767 2024-11-15T21:54:44,068 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=29 2024-11-15T21:54:44,068 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 1031049c0b4149d1189db33e38ab56cd, server=ae3311ea5ccb,37455,1731707679725 in 493 msec 2024-11-15T21:54:44,069 INFO [StoreOpener-0ab2d104e772f41ee9c56851adad43f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0ab2d104e772f41ee9c56851adad43f0 columnFamilyName colfamily11 2024-11-15T21:54:44,069 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731707682333.6694325f9c862a5a1ee02eabf7861cc3. 2024-11-15T21:54:44,069 INFO [StoreOpener-51af260ba7a9b40a7f788a5c93a416b6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 51af260ba7a9b40a7f788a5c93a416b6 columnFamilyName colfamily11 2024-11-15T21:54:44,069 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731707682333.6694325f9c862a5a1ee02eabf7861cc3. 2024-11-15T21:54:44,069 DEBUG [StoreOpener-0ab2d104e772f41ee9c56851adad43f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:44,069 DEBUG [StoreOpener-51af260ba7a9b40a7f788a5c93a416b6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:44,069 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731707682333.d9989b0341ef96818e811f874dbcf91c. 2024-11-15T21:54:44,069 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => d9989b0341ef96818e811f874dbcf91c, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731707682333.d9989b0341ef96818e811f874dbcf91c.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-15T21:54:44,070 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true d9989b0341ef96818e811f874dbcf91c 2024-11-15T21:54:44,070 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731707682333.d9989b0341ef96818e811f874dbcf91c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:44,070 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for d9989b0341ef96818e811f874dbcf91c 2024-11-15T21:54:44,070 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for d9989b0341ef96818e811f874dbcf91c 2024-11-15T21:54:44,071 INFO [StoreOpener-51af260ba7a9b40a7f788a5c93a416b6-1 {}] regionserver.HStore(327): Store=51af260ba7a9b40a7f788a5c93a416b6/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:44,071 INFO [StoreOpener-51af260ba7a9b40a7f788a5c93a416b6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 51af260ba7a9b40a7f788a5c93a416b6 2024-11-15T21:54:44,072 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=6694325f9c862a5a1ee02eabf7861cc3, regionState=OPEN, openSeqNum=2, regionLocation=ae3311ea5ccb,32871,1731707679650 2024-11-15T21:54:44,072 INFO [StoreOpener-0ab2d104e772f41ee9c56851adad43f0-1 {}] regionserver.HStore(327): Store=0ab2d104e772f41ee9c56851adad43f0/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:44,072 INFO [StoreOpener-0ab2d104e772f41ee9c56851adad43f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 0ab2d104e772f41ee9c56851adad43f0 2024-11-15T21:54:44,073 INFO [StoreOpener-d9989b0341ef96818e811f874dbcf91c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region d9989b0341ef96818e811f874dbcf91c 2024-11-15T21:54:44,075 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6694325f9c862a5a1ee02eabf7861cc3, server=ae3311ea5ccb,32871,1731707679650 because future has completed 2024-11-15T21:54:44,076 INFO [StoreOpener-51af260ba7a9b40a7f788a5c93a416b6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 51af260ba7a9b40a7f788a5c93a416b6 columnFamilyName colfamily21 2024-11-15T21:54:44,076 DEBUG [StoreOpener-51af260ba7a9b40a7f788a5c93a416b6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:44,077 INFO [StoreOpener-51af260ba7a9b40a7f788a5c93a416b6-1 {}] regionserver.HStore(327): Store=51af260ba7a9b40a7f788a5c93a416b6/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:44,077 INFO [StoreOpener-51af260ba7a9b40a7f788a5c93a416b6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 51af260ba7a9b40a7f788a5c93a416b6 2024-11-15T21:54:44,078 INFO [StoreOpener-d9989b0341ef96818e811f874dbcf91c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9989b0341ef96818e811f874dbcf91c columnFamilyName colfamily11 2024-11-15T21:54:44,078 DEBUG [StoreOpener-d9989b0341ef96818e811f874dbcf91c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:44,079 INFO [StoreOpener-0ab2d104e772f41ee9c56851adad43f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0ab2d104e772f41ee9c56851adad43f0 columnFamilyName colfamily21 2024-11-15T21:54:44,079 DEBUG [StoreOpener-0ab2d104e772f41ee9c56851adad43f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:44,080 INFO [StoreOpener-d9989b0341ef96818e811f874dbcf91c-1 {}] regionserver.HStore(327): Store=d9989b0341ef96818e811f874dbcf91c/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:44,080 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1031049c0b4149d1189db33e38ab56cd, ASSIGN in 761 msec 2024-11-15T21:54:44,080 INFO [StoreOpener-51af260ba7a9b40a7f788a5c93a416b6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 51af260ba7a9b40a7f788a5c93a416b6 columnFamilyName colfamily31 2024-11-15T21:54:44,080 DEBUG [StoreOpener-51af260ba7a9b40a7f788a5c93a416b6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:44,080 INFO [StoreOpener-d9989b0341ef96818e811f874dbcf91c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region d9989b0341ef96818e811f874dbcf91c 2024-11-15T21:54:44,081 INFO [StoreOpener-51af260ba7a9b40a7f788a5c93a416b6-1 {}] regionserver.HStore(327): Store=51af260ba7a9b40a7f788a5c93a416b6/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:44,081 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 51af260ba7a9b40a7f788a5c93a416b6 2024-11-15T21:54:44,081 INFO [StoreOpener-0ab2d104e772f41ee9c56851adad43f0-1 {}] regionserver.HStore(327): Store=0ab2d104e772f41ee9c56851adad43f0/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:44,081 INFO [StoreOpener-0ab2d104e772f41ee9c56851adad43f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 0ab2d104e772f41ee9c56851adad43f0 2024-11-15T21:54:44,082 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/51af260ba7a9b40a7f788a5c93a416b6 2024-11-15T21:54:44,084 INFO [StoreOpener-d9989b0341ef96818e811f874dbcf91c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9989b0341ef96818e811f874dbcf91c columnFamilyName colfamily21 2024-11-15T21:54:44,085 DEBUG [StoreOpener-d9989b0341ef96818e811f874dbcf91c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:44,085 INFO [StoreOpener-d9989b0341ef96818e811f874dbcf91c-1 {}] regionserver.HStore(327): Store=d9989b0341ef96818e811f874dbcf91c/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:44,085 INFO [StoreOpener-d9989b0341ef96818e811f874dbcf91c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region d9989b0341ef96818e811f874dbcf91c 2024-11-15T21:54:44,086 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/51af260ba7a9b40a7f788a5c93a416b6 2024-11-15T21:54:44,086 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=27 2024-11-15T21:54:44,086 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 6694325f9c862a5a1ee02eabf7861cc3, server=ae3311ea5ccb,32871,1731707679650 in 454 msec 2024-11-15T21:54:44,087 INFO [StoreOpener-0ab2d104e772f41ee9c56851adad43f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0ab2d104e772f41ee9c56851adad43f0 columnFamilyName colfamily31 2024-11-15T21:54:44,087 DEBUG [StoreOpener-0ab2d104e772f41ee9c56851adad43f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:44,087 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 51af260ba7a9b40a7f788a5c93a416b6 2024-11-15T21:54:44,087 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 51af260ba7a9b40a7f788a5c93a416b6 2024-11-15T21:54:44,088 INFO [StoreOpener-0ab2d104e772f41ee9c56851adad43f0-1 {}] regionserver.HStore(327): Store=0ab2d104e772f41ee9c56851adad43f0/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:44,088 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-15T21:54:44,089 INFO [StoreOpener-d9989b0341ef96818e811f874dbcf91c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9989b0341ef96818e811f874dbcf91c columnFamilyName colfamily31 2024-11-15T21:54:44,089 DEBUG [StoreOpener-d9989b0341ef96818e811f874dbcf91c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:44,089 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=6694325f9c862a5a1ee02eabf7861cc3, ASSIGN in 772 msec 2024-11-15T21:54:44,088 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 0ab2d104e772f41ee9c56851adad43f0 2024-11-15T21:54:44,089 INFO [StoreOpener-d9989b0341ef96818e811f874dbcf91c-1 {}] regionserver.HStore(327): Store=d9989b0341ef96818e811f874dbcf91c/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:44,090 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for d9989b0341ef96818e811f874dbcf91c 2024-11-15T21:54:44,090 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0ab2d104e772f41ee9c56851adad43f0 2024-11-15T21:54:44,091 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0ab2d104e772f41ee9c56851adad43f0 2024-11-15T21:54:44,091 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d9989b0341ef96818e811f874dbcf91c 2024-11-15T21:54:44,091 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d9989b0341ef96818e811f874dbcf91c 2024-11-15T21:54:44,092 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 51af260ba7a9b40a7f788a5c93a416b6 2024-11-15T21:54:44,093 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 0ab2d104e772f41ee9c56851adad43f0 2024-11-15T21:54:44,093 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 0ab2d104e772f41ee9c56851adad43f0 2024-11-15T21:54:44,093 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for d9989b0341ef96818e811f874dbcf91c 2024-11-15T21:54:44,093 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for d9989b0341ef96818e811f874dbcf91c 2024-11-15T21:54:44,094 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-15T21:54:44,095 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-15T21:54:44,095 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=13 2024-11-15T21:54:44,095 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure d28ee8a2ac06d2199c72ac7f884ba470, server=ae3311ea5ccb,40941,1731707679516 in 511 msec 2024-11-15T21:54:44,097 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 0ab2d104e772f41ee9c56851adad43f0 2024-11-15T21:54:44,097 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for d9989b0341ef96818e811f874dbcf91c 2024-11-15T21:54:44,098 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d28ee8a2ac06d2199c72ac7f884ba470, ASSIGN in 781 msec 2024-11-15T21:54:44,102 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0ab2d104e772f41ee9c56851adad43f0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T21:54:44,103 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d9989b0341ef96818e811f874dbcf91c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T21:54:44,103 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/51af260ba7a9b40a7f788a5c93a416b6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T21:54:44,103 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 0ab2d104e772f41ee9c56851adad43f0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62728181, jitterRate=-0.06527726352214813}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-15T21:54:44,103 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0ab2d104e772f41ee9c56851adad43f0 2024-11-15T21:54:44,103 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 0ab2d104e772f41ee9c56851adad43f0: Running coprocessor pre-open hook at 1731707684054Writing region info on filesystem at 1731707684054Initializing all the Stores at 1731707684062 (+8 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707684062Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707684063 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707684063Cleaning up temporary data from old regions at 1731707684093 (+30 ms)Running coprocessor post-open hooks at 1731707684103 (+10 ms)Region opened successfully at 1731707684103 2024-11-15T21:54:44,104 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened d9989b0341ef96818e811f874dbcf91c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67322023, jitterRate=0.0031763166189193726}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-15T21:54:44,104 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 51af260ba7a9b40a7f788a5c93a416b6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61421613, jitterRate=-0.08474664390087128}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-15T21:54:44,104 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d9989b0341ef96818e811f874dbcf91c 2024-11-15T21:54:44,104 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 51af260ba7a9b40a7f788a5c93a416b6 2024-11-15T21:54:44,104 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for d9989b0341ef96818e811f874dbcf91c: Running coprocessor pre-open hook at 1731707684070Writing region info on filesystem at 1731707684070Initializing all the Stores at 1731707684072 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707684072Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707684073 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707684073Cleaning up temporary data from old regions at 1731707684093 (+20 ms)Running coprocessor post-open hooks at 1731707684104 (+11 ms)Region opened successfully at 1731707684104 2024-11-15T21:54:44,104 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 51af260ba7a9b40a7f788a5c93a416b6: Running coprocessor pre-open hook at 1731707684050Writing region info on filesystem at 1731707684050Initializing all the Stores at 1731707684062 (+12 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707684062Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707684063 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707684063Cleaning up temporary data from old regions at 1731707684087 (+24 ms)Running coprocessor post-open hooks at 1731707684104 (+17 ms)Region opened successfully at 1731707684104 2024-11-15T21:54:44,105 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731707682333.0ab2d104e772f41ee9c56851adad43f0., pid=32, masterSystemTime=1731707683657 2024-11-15T21:54:44,106 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731707682333.d9989b0341ef96818e811f874dbcf91c., pid=48, masterSystemTime=1731707683767 2024-11-15T21:54:44,106 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731707682333.51af260ba7a9b40a7f788a5c93a416b6., pid=37, masterSystemTime=1731707683686 2024-11-15T21:54:44,109 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731707682333.0ab2d104e772f41ee9c56851adad43f0. 2024-11-15T21:54:44,109 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731707682333.0ab2d104e772f41ee9c56851adad43f0. 2024-11-15T21:54:44,109 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731707682333.3f13d1331edcf64c426c3bf9433330e4. 2024-11-15T21:54:44,110 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=0ab2d104e772f41ee9c56851adad43f0, regionState=OPEN, openSeqNum=2, regionLocation=ae3311ea5ccb,40941,1731707679516 2024-11-15T21:54:44,110 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 3f13d1331edcf64c426c3bf9433330e4, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731707682333.3f13d1331edcf64c426c3bf9433330e4.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-15T21:54:44,110 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 3f13d1331edcf64c426c3bf9433330e4 2024-11-15T21:54:44,110 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731707682333.3f13d1331edcf64c426c3bf9433330e4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:44,110 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 3f13d1331edcf64c426c3bf9433330e4 2024-11-15T21:54:44,110 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 3f13d1331edcf64c426c3bf9433330e4 2024-11-15T21:54:44,111 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731707682333.51af260ba7a9b40a7f788a5c93a416b6. 2024-11-15T21:54:44,111 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731707682333.51af260ba7a9b40a7f788a5c93a416b6. 2024-11-15T21:54:44,112 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731707682333.fac6af87aa0fc9826df0515f766c031e. 2024-11-15T21:54:44,112 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => fac6af87aa0fc9826df0515f766c031e, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731707682333.fac6af87aa0fc9826df0515f766c031e.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-15T21:54:44,112 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true fac6af87aa0fc9826df0515f766c031e 2024-11-15T21:54:44,113 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731707682333.fac6af87aa0fc9826df0515f766c031e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:44,113 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for fac6af87aa0fc9826df0515f766c031e 2024-11-15T21:54:44,113 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for fac6af87aa0fc9826df0515f766c031e 2024-11-15T21:54:44,113 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0ab2d104e772f41ee9c56851adad43f0, server=ae3311ea5ccb,40941,1731707679516 because future has completed 2024-11-15T21:54:44,113 INFO [StoreOpener-3f13d1331edcf64c426c3bf9433330e4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 3f13d1331edcf64c426c3bf9433330e4 2024-11-15T21:54:44,113 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731707682333.d9989b0341ef96818e811f874dbcf91c. 2024-11-15T21:54:44,114 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731707682333.d9989b0341ef96818e811f874dbcf91c. 2024-11-15T21:54:44,114 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731707682333.3cf264afea186a5cc96edd3e68a10d45. 2024-11-15T21:54:44,114 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => 3cf264afea186a5cc96edd3e68a10d45, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731707682333.3cf264afea186a5cc96edd3e68a10d45.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-15T21:54:44,114 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 3cf264afea186a5cc96edd3e68a10d45 2024-11-15T21:54:44,114 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731707682333.3cf264afea186a5cc96edd3e68a10d45.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:44,115 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for 3cf264afea186a5cc96edd3e68a10d45 2024-11-15T21:54:44,115 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for 3cf264afea186a5cc96edd3e68a10d45 2024-11-15T21:54:44,116 INFO [StoreOpener-3f13d1331edcf64c426c3bf9433330e4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3f13d1331edcf64c426c3bf9433330e4 columnFamilyName colfamily11 2024-11-15T21:54:44,116 DEBUG [StoreOpener-3f13d1331edcf64c426c3bf9433330e4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:44,117 INFO [StoreOpener-3cf264afea186a5cc96edd3e68a10d45-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 3cf264afea186a5cc96edd3e68a10d45 2024-11-15T21:54:44,117 INFO [StoreOpener-3f13d1331edcf64c426c3bf9433330e4-1 {}] regionserver.HStore(327): Store=3f13d1331edcf64c426c3bf9433330e4/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:44,118 INFO [StoreOpener-3f13d1331edcf64c426c3bf9433330e4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 3f13d1331edcf64c426c3bf9433330e4 2024-11-15T21:54:44,118 INFO [StoreOpener-fac6af87aa0fc9826df0515f766c031e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region fac6af87aa0fc9826df0515f766c031e 2024-11-15T21:54:44,120 INFO [StoreOpener-3f13d1331edcf64c426c3bf9433330e4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3f13d1331edcf64c426c3bf9433330e4 columnFamilyName colfamily21 2024-11-15T21:54:44,120 DEBUG [StoreOpener-3f13d1331edcf64c426c3bf9433330e4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:44,120 INFO [StoreOpener-3cf264afea186a5cc96edd3e68a10d45-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3cf264afea186a5cc96edd3e68a10d45 columnFamilyName colfamily11 2024-11-15T21:54:44,120 DEBUG [StoreOpener-3cf264afea186a5cc96edd3e68a10d45-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:44,120 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=19 2024-11-15T21:54:44,120 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure 0ab2d104e772f41ee9c56851adad43f0, server=ae3311ea5ccb,40941,1731707679516 in 601 msec 2024-11-15T21:54:44,121 INFO [StoreOpener-3cf264afea186a5cc96edd3e68a10d45-1 {}] regionserver.HStore(327): Store=3cf264afea186a5cc96edd3e68a10d45/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:44,121 INFO [StoreOpener-3cf264afea186a5cc96edd3e68a10d45-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 3cf264afea186a5cc96edd3e68a10d45 2024-11-15T21:54:44,121 INFO [StoreOpener-3f13d1331edcf64c426c3bf9433330e4-1 {}] regionserver.HStore(327): Store=3f13d1331edcf64c426c3bf9433330e4/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:44,121 INFO [StoreOpener-3f13d1331edcf64c426c3bf9433330e4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 3f13d1331edcf64c426c3bf9433330e4 2024-11-15T21:54:44,123 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=0ab2d104e772f41ee9c56851adad43f0, ASSIGN in 806 msec 2024-11-15T21:54:44,123 INFO [StoreOpener-3cf264afea186a5cc96edd3e68a10d45-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3cf264afea186a5cc96edd3e68a10d45 columnFamilyName colfamily21 2024-11-15T21:54:44,124 DEBUG [StoreOpener-3cf264afea186a5cc96edd3e68a10d45-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:44,124 INFO [StoreOpener-fac6af87aa0fc9826df0515f766c031e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fac6af87aa0fc9826df0515f766c031e columnFamilyName colfamily11 2024-11-15T21:54:44,124 DEBUG [StoreOpener-fac6af87aa0fc9826df0515f766c031e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:44,124 INFO [StoreOpener-3cf264afea186a5cc96edd3e68a10d45-1 {}] regionserver.HStore(327): Store=3cf264afea186a5cc96edd3e68a10d45/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:44,124 INFO [StoreOpener-3f13d1331edcf64c426c3bf9433330e4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3f13d1331edcf64c426c3bf9433330e4 columnFamilyName colfamily31 2024-11-15T21:54:44,124 INFO [StoreOpener-3cf264afea186a5cc96edd3e68a10d45-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 3cf264afea186a5cc96edd3e68a10d45 2024-11-15T21:54:44,125 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=d9989b0341ef96818e811f874dbcf91c, regionState=OPEN, openSeqNum=2, regionLocation=ae3311ea5ccb,32871,1731707679650 2024-11-15T21:54:44,126 DEBUG [StoreOpener-3f13d1331edcf64c426c3bf9433330e4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:44,126 INFO [StoreOpener-3cf264afea186a5cc96edd3e68a10d45-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3cf264afea186a5cc96edd3e68a10d45 columnFamilyName colfamily31 2024-11-15T21:54:44,126 DEBUG [StoreOpener-3cf264afea186a5cc96edd3e68a10d45-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:44,127 INFO [StoreOpener-3f13d1331edcf64c426c3bf9433330e4-1 {}] regionserver.HStore(327): Store=3f13d1331edcf64c426c3bf9433330e4/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:44,127 INFO [StoreOpener-3cf264afea186a5cc96edd3e68a10d45-1 {}] regionserver.HStore(327): Store=3cf264afea186a5cc96edd3e68a10d45/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:44,127 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 3f13d1331edcf64c426c3bf9433330e4 2024-11-15T21:54:44,127 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for 3cf264afea186a5cc96edd3e68a10d45 2024-11-15T21:54:44,128 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure d9989b0341ef96818e811f874dbcf91c, server=ae3311ea5ccb,32871,1731707679650 because future has completed 2024-11-15T21:54:44,129 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3f13d1331edcf64c426c3bf9433330e4 2024-11-15T21:54:44,129 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3cf264afea186a5cc96edd3e68a10d45 2024-11-15T21:54:44,129 INFO [StoreOpener-fac6af87aa0fc9826df0515f766c031e-1 {}] regionserver.HStore(327): Store=fac6af87aa0fc9826df0515f766c031e/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:44,129 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3f13d1331edcf64c426c3bf9433330e4 2024-11-15T21:54:44,129 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3cf264afea186a5cc96edd3e68a10d45 2024-11-15T21:54:44,129 INFO [StoreOpener-fac6af87aa0fc9826df0515f766c031e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region fac6af87aa0fc9826df0515f766c031e 2024-11-15T21:54:44,130 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=51af260ba7a9b40a7f788a5c93a416b6, regionState=OPEN, openSeqNum=2, regionLocation=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:54:44,133 INFO [StoreOpener-fac6af87aa0fc9826df0515f766c031e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fac6af87aa0fc9826df0515f766c031e columnFamilyName colfamily21 2024-11-15T21:54:44,133 DEBUG [StoreOpener-fac6af87aa0fc9826df0515f766c031e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:44,134 INFO [StoreOpener-fac6af87aa0fc9826df0515f766c031e-1 {}] regionserver.HStore(327): Store=fac6af87aa0fc9826df0515f766c031e/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:44,134 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for 3cf264afea186a5cc96edd3e68a10d45 2024-11-15T21:54:44,134 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 51af260ba7a9b40a7f788a5c93a416b6, server=ae3311ea5ccb,37455,1731707679725 because future has completed 2024-11-15T21:54:44,134 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for 3cf264afea186a5cc96edd3e68a10d45 2024-11-15T21:54:44,134 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 3f13d1331edcf64c426c3bf9433330e4 2024-11-15T21:54:44,135 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 3f13d1331edcf64c426c3bf9433330e4 2024-11-15T21:54:44,135 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36531 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=ae3311ea5ccb,37455,1731707679725, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=51af260ba7a9b40a7f788a5c93a416b6. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T21:54:44,136 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-15T21:54:44,136 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-15T21:54:44,137 INFO [StoreOpener-fac6af87aa0fc9826df0515f766c031e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region fac6af87aa0fc9826df0515f766c031e 2024-11-15T21:54:44,138 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for 3cf264afea186a5cc96edd3e68a10d45 2024-11-15T21:54:44,138 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 3f13d1331edcf64c426c3bf9433330e4 2024-11-15T21:54:44,140 INFO [StoreOpener-fac6af87aa0fc9826df0515f766c031e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fac6af87aa0fc9826df0515f766c031e columnFamilyName colfamily31 2024-11-15T21:54:44,140 DEBUG [StoreOpener-fac6af87aa0fc9826df0515f766c031e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:44,142 INFO [StoreOpener-fac6af87aa0fc9826df0515f766c031e-1 {}] regionserver.HStore(327): Store=fac6af87aa0fc9826df0515f766c031e/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:44,142 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for fac6af87aa0fc9826df0515f766c031e 2024-11-15T21:54:44,142 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3cf264afea186a5cc96edd3e68a10d45/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T21:54:44,143 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=22 2024-11-15T21:54:44,143 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure d9989b0341ef96818e811f874dbcf91c, server=ae3311ea5ccb,32871,1731707679650 in 518 msec 2024-11-15T21:54:44,143 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened 3cf264afea186a5cc96edd3e68a10d45; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61192420, jitterRate=-0.0881618857383728}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-15T21:54:44,143 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3cf264afea186a5cc96edd3e68a10d45 2024-11-15T21:54:44,143 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for 3cf264afea186a5cc96edd3e68a10d45: Running coprocessor pre-open hook at 1731707684115Writing region info on filesystem at 1731707684115Initializing all the Stores at 1731707684116 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707684116Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707684117 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707684117Cleaning up temporary data from old regions at 1731707684134 (+17 ms)Running coprocessor post-open hooks at 1731707684143 (+9 ms)Region opened successfully at 1731707684143 2024-11-15T21:54:44,144 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fac6af87aa0fc9826df0515f766c031e 2024-11-15T21:54:44,144 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3f13d1331edcf64c426c3bf9433330e4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T21:54:44,145 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fac6af87aa0fc9826df0515f766c031e 2024-11-15T21:54:44,145 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 3f13d1331edcf64c426c3bf9433330e4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64000711, jitterRate=-0.046315088868141174}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-15T21:54:44,145 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3f13d1331edcf64c426c3bf9433330e4 2024-11-15T21:54:44,146 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731707682333.3cf264afea186a5cc96edd3e68a10d45., pid=47, masterSystemTime=1731707683767 2024-11-15T21:54:44,146 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 3f13d1331edcf64c426c3bf9433330e4: Running coprocessor pre-open hook at 1731707684111Writing region info on filesystem at 1731707684111Initializing all the Stores at 1731707684113 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707684113Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707684113Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707684113Cleaning up temporary data from old regions at 1731707684135 (+22 ms)Running coprocessor post-open hooks at 1731707684145 (+10 ms)Region opened successfully at 1731707684146 (+1 ms) 2024-11-15T21:54:44,146 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d9989b0341ef96818e811f874dbcf91c, ASSIGN in 829 msec 2024-11-15T21:54:44,147 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for fac6af87aa0fc9826df0515f766c031e 2024-11-15T21:54:44,147 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for fac6af87aa0fc9826df0515f766c031e 2024-11-15T21:54:44,148 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731707682333.3f13d1331edcf64c426c3bf9433330e4., pid=33, masterSystemTime=1731707683657 2024-11-15T21:54:44,148 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-15T21:54:44,148 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731707682333.3cf264afea186a5cc96edd3e68a10d45. 2024-11-15T21:54:44,148 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731707682333.3cf264afea186a5cc96edd3e68a10d45. 2024-11-15T21:54:44,149 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731707682333.4256fff2fa33299a6264dd460a4003fb. 2024-11-15T21:54:44,149 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => 4256fff2fa33299a6264dd460a4003fb, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731707682333.4256fff2fa33299a6264dd460a4003fb.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-15T21:54:44,150 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 4256fff2fa33299a6264dd460a4003fb 2024-11-15T21:54:44,150 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731707682333.4256fff2fa33299a6264dd460a4003fb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:44,150 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for 4256fff2fa33299a6264dd460a4003fb 2024-11-15T21:54:44,150 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for 4256fff2fa33299a6264dd460a4003fb 2024-11-15T21:54:44,152 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731707682333.3f13d1331edcf64c426c3bf9433330e4. 2024-11-15T21:54:44,152 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731707682333.3f13d1331edcf64c426c3bf9433330e4. 2024-11-15T21:54:44,153 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for fac6af87aa0fc9826df0515f766c031e 2024-11-15T21:54:44,153 INFO [StoreOpener-4256fff2fa33299a6264dd460a4003fb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 4256fff2fa33299a6264dd460a4003fb 2024-11-15T21:54:44,154 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=18 2024-11-15T21:54:44,154 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure 51af260ba7a9b40a7f788a5c93a416b6, server=ae3311ea5ccb,37455,1731707679725 in 598 msec 2024-11-15T21:54:44,157 INFO [StoreOpener-4256fff2fa33299a6264dd460a4003fb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4256fff2fa33299a6264dd460a4003fb columnFamilyName colfamily11 2024-11-15T21:54:44,157 DEBUG [StoreOpener-4256fff2fa33299a6264dd460a4003fb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:44,157 INFO [StoreOpener-4256fff2fa33299a6264dd460a4003fb-1 {}] regionserver.HStore(327): Store=4256fff2fa33299a6264dd460a4003fb/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:44,158 INFO [StoreOpener-4256fff2fa33299a6264dd460a4003fb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 4256fff2fa33299a6264dd460a4003fb 2024-11-15T21:54:44,158 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=51af260ba7a9b40a7f788a5c93a416b6, ASSIGN in 840 msec 2024-11-15T21:54:44,158 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fac6af87aa0fc9826df0515f766c031e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T21:54:44,159 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened fac6af87aa0fc9826df0515f766c031e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58970774, jitterRate=-0.12126699090003967}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-15T21:54:44,159 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fac6af87aa0fc9826df0515f766c031e 2024-11-15T21:54:44,159 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for fac6af87aa0fc9826df0515f766c031e: Running coprocessor pre-open hook at 1731707684114Writing region info on filesystem at 1731707684114Initializing all the Stores at 1731707684116 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707684116Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707684118 (+2 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707684118Cleaning up temporary data from old regions at 1731707684147 (+29 ms)Running coprocessor post-open hooks at 1731707684159 (+12 ms)Region opened successfully at 1731707684159 2024-11-15T21:54:44,160 INFO [StoreOpener-4256fff2fa33299a6264dd460a4003fb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4256fff2fa33299a6264dd460a4003fb columnFamilyName colfamily21 2024-11-15T21:54:44,160 DEBUG [StoreOpener-4256fff2fa33299a6264dd460a4003fb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:44,161 INFO [StoreOpener-4256fff2fa33299a6264dd460a4003fb-1 {}] regionserver.HStore(327): Store=4256fff2fa33299a6264dd460a4003fb/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:44,161 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=3f13d1331edcf64c426c3bf9433330e4, regionState=OPEN, openSeqNum=2, regionLocation=ae3311ea5ccb,40941,1731707679516 2024-11-15T21:54:44,161 INFO [StoreOpener-4256fff2fa33299a6264dd460a4003fb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 4256fff2fa33299a6264dd460a4003fb 2024-11-15T21:54:44,161 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731707682333.fac6af87aa0fc9826df0515f766c031e., pid=34, masterSystemTime=1731707683686 2024-11-15T21:54:44,164 INFO [StoreOpener-4256fff2fa33299a6264dd460a4003fb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4256fff2fa33299a6264dd460a4003fb columnFamilyName colfamily31 2024-11-15T21:54:44,164 DEBUG [StoreOpener-4256fff2fa33299a6264dd460a4003fb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:44,165 INFO [StoreOpener-4256fff2fa33299a6264dd460a4003fb-1 {}] regionserver.HStore(327): Store=4256fff2fa33299a6264dd460a4003fb/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:44,165 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3f13d1331edcf64c426c3bf9433330e4, server=ae3311ea5ccb,40941,1731707679516 because future has completed 2024-11-15T21:54:44,165 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for 4256fff2fa33299a6264dd460a4003fb 2024-11-15T21:54:44,166 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731707682333.fac6af87aa0fc9826df0515f766c031e. 2024-11-15T21:54:44,166 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731707682333.fac6af87aa0fc9826df0515f766c031e. 2024-11-15T21:54:44,166 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4256fff2fa33299a6264dd460a4003fb 2024-11-15T21:54:44,166 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731707682333.50df10b3ae47c5dc7c4521c4c58c192f. 2024-11-15T21:54:44,166 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=3cf264afea186a5cc96edd3e68a10d45, regionState=OPEN, openSeqNum=2, regionLocation=ae3311ea5ccb,32871,1731707679650 2024-11-15T21:54:44,166 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => 50df10b3ae47c5dc7c4521c4c58c192f, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731707682333.50df10b3ae47c5dc7c4521c4c58c192f.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-15T21:54:44,166 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 50df10b3ae47c5dc7c4521c4c58c192f 2024-11-15T21:54:44,166 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4256fff2fa33299a6264dd460a4003fb 2024-11-15T21:54:44,167 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731707682333.50df10b3ae47c5dc7c4521c4c58c192f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:44,167 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for 50df10b3ae47c5dc7c4521c4c58c192f 2024-11-15T21:54:44,167 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for 50df10b3ae47c5dc7c4521c4c58c192f 2024-11-15T21:54:44,169 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for 4256fff2fa33299a6264dd460a4003fb 2024-11-15T21:54:44,169 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for 4256fff2fa33299a6264dd460a4003fb 2024-11-15T21:54:44,170 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36531 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=ae3311ea5ccb,32871,1731707679650, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3cf264afea186a5cc96edd3e68a10d45. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T21:54:44,170 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-15T21:54:44,170 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3cf264afea186a5cc96edd3e68a10d45, server=ae3311ea5ccb,32871,1731707679650 because future has completed 2024-11-15T21:54:44,172 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for 4256fff2fa33299a6264dd460a4003fb 2024-11-15T21:54:44,176 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=20 2024-11-15T21:54:44,176 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 3cf264afea186a5cc96edd3e68a10d45, server=ae3311ea5ccb,32871,1731707679650 in 558 msec 2024-11-15T21:54:44,178 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3cf264afea186a5cc96edd3e68a10d45, ASSIGN in 862 msec 2024-11-15T21:54:44,179 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=fac6af87aa0fc9826df0515f766c031e, regionState=OPEN, openSeqNum=2, regionLocation=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:54:44,182 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure fac6af87aa0fc9826df0515f766c031e, server=ae3311ea5ccb,37455,1731707679725 because future has completed 2024-11-15T21:54:44,185 INFO [StoreOpener-50df10b3ae47c5dc7c4521c4c58c192f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 50df10b3ae47c5dc7c4521c4c58c192f 2024-11-15T21:54:44,186 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=10 2024-11-15T21:54:44,186 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 3f13d1331edcf64c426c3bf9433330e4, server=ae3311ea5ccb,40941,1731707679516 in 658 msec 2024-11-15T21:54:44,189 INFO [StoreOpener-50df10b3ae47c5dc7c4521c4c58c192f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 50df10b3ae47c5dc7c4521c4c58c192f columnFamilyName colfamily11 2024-11-15T21:54:44,189 DEBUG [StoreOpener-50df10b3ae47c5dc7c4521c4c58c192f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:44,189 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3f13d1331edcf64c426c3bf9433330e4, ASSIGN in 872 msec 2024-11-15T21:54:44,190 INFO [StoreOpener-50df10b3ae47c5dc7c4521c4c58c192f-1 {}] regionserver.HStore(327): Store=50df10b3ae47c5dc7c4521c4c58c192f/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:44,190 INFO [StoreOpener-50df10b3ae47c5dc7c4521c4c58c192f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 50df10b3ae47c5dc7c4521c4c58c192f 2024-11-15T21:54:44,192 INFO [StoreOpener-50df10b3ae47c5dc7c4521c4c58c192f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 50df10b3ae47c5dc7c4521c4c58c192f columnFamilyName colfamily21 2024-11-15T21:54:44,192 DEBUG [StoreOpener-50df10b3ae47c5dc7c4521c4c58c192f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:44,193 INFO [StoreOpener-50df10b3ae47c5dc7c4521c4c58c192f-1 {}] regionserver.HStore(327): Store=50df10b3ae47c5dc7c4521c4c58c192f/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:44,193 INFO [StoreOpener-50df10b3ae47c5dc7c4521c4c58c192f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 50df10b3ae47c5dc7c4521c4c58c192f 2024-11-15T21:54:44,194 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=25 2024-11-15T21:54:44,194 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure fac6af87aa0fc9826df0515f766c031e, server=ae3311ea5ccb,37455,1731707679725 in 658 msec 2024-11-15T21:54:44,194 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4256fff2fa33299a6264dd460a4003fb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T21:54:44,195 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened 4256fff2fa33299a6264dd460a4003fb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69637744, jitterRate=0.03768324851989746}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-15T21:54:44,195 INFO [StoreOpener-50df10b3ae47c5dc7c4521c4c58c192f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 50df10b3ae47c5dc7c4521c4c58c192f columnFamilyName colfamily31 2024-11-15T21:54:44,195 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4256fff2fa33299a6264dd460a4003fb 2024-11-15T21:54:44,195 DEBUG [StoreOpener-50df10b3ae47c5dc7c4521c4c58c192f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:44,195 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for 4256fff2fa33299a6264dd460a4003fb: Running coprocessor pre-open hook at 1731707684150Writing region info on filesystem at 1731707684150Initializing all the Stores at 1731707684153 (+3 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707684153Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707684153Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707684153Cleaning up temporary data from old regions at 1731707684169 (+16 ms)Running coprocessor post-open hooks at 1731707684195 (+26 ms)Region opened successfully at 1731707684195 2024-11-15T21:54:44,196 INFO [StoreOpener-50df10b3ae47c5dc7c4521c4c58c192f-1 {}] regionserver.HStore(327): Store=50df10b3ae47c5dc7c4521c4c58c192f/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:44,196 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for 50df10b3ae47c5dc7c4521c4c58c192f 2024-11-15T21:54:44,197 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731707682333.4256fff2fa33299a6264dd460a4003fb., pid=53, masterSystemTime=1731707683767 2024-11-15T21:54:44,197 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/50df10b3ae47c5dc7c4521c4c58c192f 2024-11-15T21:54:44,198 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/50df10b3ae47c5dc7c4521c4c58c192f 2024-11-15T21:54:44,199 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fac6af87aa0fc9826df0515f766c031e, ASSIGN in 880 msec 2024-11-15T21:54:44,200 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for 50df10b3ae47c5dc7c4521c4c58c192f 2024-11-15T21:54:44,200 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for 50df10b3ae47c5dc7c4521c4c58c192f 2024-11-15T21:54:44,200 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731707682333.4256fff2fa33299a6264dd460a4003fb. 2024-11-15T21:54:44,200 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731707682333.4256fff2fa33299a6264dd460a4003fb. 2024-11-15T21:54:44,200 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731707682333.aec2e46cb64898ba2a66825d3732f6dc. 2024-11-15T21:54:44,201 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-15T21:54:44,201 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => aec2e46cb64898ba2a66825d3732f6dc, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731707682333.aec2e46cb64898ba2a66825d3732f6dc.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-15T21:54:44,201 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true aec2e46cb64898ba2a66825d3732f6dc 2024-11-15T21:54:44,201 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731707682333.aec2e46cb64898ba2a66825d3732f6dc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:44,201 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for aec2e46cb64898ba2a66825d3732f6dc 2024-11-15T21:54:44,201 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for aec2e46cb64898ba2a66825d3732f6dc 2024-11-15T21:54:44,202 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=4256fff2fa33299a6264dd460a4003fb, regionState=OPEN, openSeqNum=2, regionLocation=ae3311ea5ccb,32871,1731707679650 2024-11-15T21:54:44,202 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for 50df10b3ae47c5dc7c4521c4c58c192f 2024-11-15T21:54:44,206 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4256fff2fa33299a6264dd460a4003fb, server=ae3311ea5ccb,32871,1731707679650 because future has completed 2024-11-15T21:54:44,206 INFO [StoreOpener-aec2e46cb64898ba2a66825d3732f6dc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region aec2e46cb64898ba2a66825d3732f6dc 2024-11-15T21:54:44,209 INFO [StoreOpener-aec2e46cb64898ba2a66825d3732f6dc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aec2e46cb64898ba2a66825d3732f6dc columnFamilyName colfamily11 2024-11-15T21:54:44,209 DEBUG [StoreOpener-aec2e46cb64898ba2a66825d3732f6dc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:44,210 INFO [StoreOpener-aec2e46cb64898ba2a66825d3732f6dc-1 {}] regionserver.HStore(327): Store=aec2e46cb64898ba2a66825d3732f6dc/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:44,210 INFO [StoreOpener-aec2e46cb64898ba2a66825d3732f6dc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region aec2e46cb64898ba2a66825d3732f6dc 2024-11-15T21:54:44,211 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=8 2024-11-15T21:54:44,211 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 4256fff2fa33299a6264dd460a4003fb, server=ae3311ea5ccb,32871,1731707679650 in 557 msec 2024-11-15T21:54:44,212 INFO [StoreOpener-aec2e46cb64898ba2a66825d3732f6dc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aec2e46cb64898ba2a66825d3732f6dc columnFamilyName colfamily21 2024-11-15T21:54:44,212 DEBUG [StoreOpener-aec2e46cb64898ba2a66825d3732f6dc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:44,212 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4256fff2fa33299a6264dd460a4003fb, ASSIGN in 897 msec 2024-11-15T21:54:44,212 INFO [StoreOpener-aec2e46cb64898ba2a66825d3732f6dc-1 {}] regionserver.HStore(327): Store=aec2e46cb64898ba2a66825d3732f6dc/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:44,213 INFO [StoreOpener-aec2e46cb64898ba2a66825d3732f6dc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region aec2e46cb64898ba2a66825d3732f6dc 2024-11-15T21:54:44,214 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/50df10b3ae47c5dc7c4521c4c58c192f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T21:54:44,214 INFO [StoreOpener-aec2e46cb64898ba2a66825d3732f6dc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aec2e46cb64898ba2a66825d3732f6dc columnFamilyName colfamily31 2024-11-15T21:54:44,215 DEBUG [StoreOpener-aec2e46cb64898ba2a66825d3732f6dc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:44,215 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened 50df10b3ae47c5dc7c4521c4c58c192f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62604321, jitterRate=-0.0671229213476181}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-15T21:54:44,215 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 50df10b3ae47c5dc7c4521c4c58c192f 2024-11-15T21:54:44,215 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for 50df10b3ae47c5dc7c4521c4c58c192f: Running coprocessor pre-open hook at 1731707684167Writing region info on filesystem at 1731707684167Initializing all the Stores at 1731707684168 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707684168Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707684185 (+17 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707684185Cleaning up temporary data from old regions at 1731707684200 (+15 ms)Running coprocessor post-open hooks at 1731707684215 (+15 ms)Region opened successfully at 1731707684215 2024-11-15T21:54:44,215 INFO [StoreOpener-aec2e46cb64898ba2a66825d3732f6dc-1 {}] regionserver.HStore(327): Store=aec2e46cb64898ba2a66825d3732f6dc/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:44,216 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for aec2e46cb64898ba2a66825d3732f6dc 2024-11-15T21:54:44,216 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aec2e46cb64898ba2a66825d3732f6dc 2024-11-15T21:54:44,217 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731707682333.50df10b3ae47c5dc7c4521c4c58c192f., pid=51, masterSystemTime=1731707683686 2024-11-15T21:54:44,217 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aec2e46cb64898ba2a66825d3732f6dc 2024-11-15T21:54:44,220 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for aec2e46cb64898ba2a66825d3732f6dc 2024-11-15T21:54:44,220 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for aec2e46cb64898ba2a66825d3732f6dc 2024-11-15T21:54:44,220 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731707682333.50df10b3ae47c5dc7c4521c4c58c192f. 2024-11-15T21:54:44,220 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731707682333.50df10b3ae47c5dc7c4521c4c58c192f. 2024-11-15T21:54:44,220 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-15T21:54:44,221 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=50df10b3ae47c5dc7c4521c4c58c192f, regionState=OPEN, openSeqNum=2, regionLocation=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:54:44,222 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for aec2e46cb64898ba2a66825d3732f6dc 2024-11-15T21:54:44,224 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 50df10b3ae47c5dc7c4521c4c58c192f, server=ae3311ea5ccb,37455,1731707679725 because future has completed 2024-11-15T21:54:44,228 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aec2e46cb64898ba2a66825d3732f6dc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T21:54:44,229 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened aec2e46cb64898ba2a66825d3732f6dc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62297211, jitterRate=-0.07169921696186066}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-15T21:54:44,229 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for aec2e46cb64898ba2a66825d3732f6dc 2024-11-15T21:54:44,229 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for aec2e46cb64898ba2a66825d3732f6dc: Running coprocessor pre-open hook at 1731707684202Writing region info on filesystem at 1731707684202Initializing all the Stores at 1731707684203 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707684203Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707684206 (+3 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707684206Cleaning up temporary data from old regions at 1731707684220 (+14 ms)Running coprocessor post-open hooks at 1731707684229 (+9 ms)Region opened successfully at 1731707684229 2024-11-15T21:54:44,230 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=7 2024-11-15T21:54:44,230 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 50df10b3ae47c5dc7c4521c4c58c192f, server=ae3311ea5ccb,37455,1731707679725 in 593 msec 2024-11-15T21:54:44,230 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731707682333.aec2e46cb64898ba2a66825d3732f6dc., pid=55, masterSystemTime=1731707683767 2024-11-15T21:54:44,233 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=50df10b3ae47c5dc7c4521c4c58c192f, ASSIGN in 916 msec 2024-11-15T21:54:44,233 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731707682333.aec2e46cb64898ba2a66825d3732f6dc. 2024-11-15T21:54:44,233 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731707682333.aec2e46cb64898ba2a66825d3732f6dc. 2024-11-15T21:54:44,234 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731707682333.dd723478fcfc0e25b606965c9135244b. 2024-11-15T21:54:44,234 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => dd723478fcfc0e25b606965c9135244b, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731707682333.dd723478fcfc0e25b606965c9135244b.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-15T21:54:44,234 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true dd723478fcfc0e25b606965c9135244b 2024-11-15T21:54:44,234 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731707682333.dd723478fcfc0e25b606965c9135244b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:54:44,234 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for dd723478fcfc0e25b606965c9135244b 2024-11-15T21:54:44,234 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for dd723478fcfc0e25b606965c9135244b 2024-11-15T21:54:44,235 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=aec2e46cb64898ba2a66825d3732f6dc, regionState=OPEN, openSeqNum=2, regionLocation=ae3311ea5ccb,32871,1731707679650 2024-11-15T21:54:44,243 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure aec2e46cb64898ba2a66825d3732f6dc, server=ae3311ea5ccb,32871,1731707679650 because future has completed 2024-11-15T21:54:44,249 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=14 2024-11-15T21:54:44,249 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure aec2e46cb64898ba2a66825d3732f6dc, server=ae3311ea5ccb,32871,1731707679650 in 585 msec 2024-11-15T21:54:44,251 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=aec2e46cb64898ba2a66825d3732f6dc, ASSIGN in 935 msec 2024-11-15T21:54:44,253 INFO [StoreOpener-dd723478fcfc0e25b606965c9135244b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region dd723478fcfc0e25b606965c9135244b 2024-11-15T21:54:44,256 INFO [StoreOpener-dd723478fcfc0e25b606965c9135244b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dd723478fcfc0e25b606965c9135244b columnFamilyName colfamily11 2024-11-15T21:54:44,256 DEBUG [StoreOpener-dd723478fcfc0e25b606965c9135244b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:44,257 INFO [StoreOpener-dd723478fcfc0e25b606965c9135244b-1 {}] regionserver.HStore(327): Store=dd723478fcfc0e25b606965c9135244b/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:44,257 INFO [StoreOpener-dd723478fcfc0e25b606965c9135244b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region dd723478fcfc0e25b606965c9135244b 2024-11-15T21:54:44,259 INFO [StoreOpener-dd723478fcfc0e25b606965c9135244b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dd723478fcfc0e25b606965c9135244b columnFamilyName colfamily21 2024-11-15T21:54:44,259 DEBUG [StoreOpener-dd723478fcfc0e25b606965c9135244b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:44,261 INFO [StoreOpener-dd723478fcfc0e25b606965c9135244b-1 {}] regionserver.HStore(327): Store=dd723478fcfc0e25b606965c9135244b/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:44,261 INFO [StoreOpener-dd723478fcfc0e25b606965c9135244b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region dd723478fcfc0e25b606965c9135244b 2024-11-15T21:54:44,263 INFO [StoreOpener-dd723478fcfc0e25b606965c9135244b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dd723478fcfc0e25b606965c9135244b columnFamilyName colfamily31 2024-11-15T21:54:44,263 DEBUG [StoreOpener-dd723478fcfc0e25b606965c9135244b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:54:44,263 INFO [StoreOpener-dd723478fcfc0e25b606965c9135244b-1 {}] regionserver.HStore(327): Store=dd723478fcfc0e25b606965c9135244b/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:54:44,264 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for dd723478fcfc0e25b606965c9135244b 2024-11-15T21:54:44,265 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dd723478fcfc0e25b606965c9135244b 2024-11-15T21:54:44,265 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dd723478fcfc0e25b606965c9135244b 2024-11-15T21:54:44,266 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for dd723478fcfc0e25b606965c9135244b 2024-11-15T21:54:44,266 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for dd723478fcfc0e25b606965c9135244b 2024-11-15T21:54:44,267 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-15T21:54:44,268 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for dd723478fcfc0e25b606965c9135244b 2024-11-15T21:54:44,271 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dd723478fcfc0e25b606965c9135244b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T21:54:44,272 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened dd723478fcfc0e25b606965c9135244b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63665776, jitterRate=-0.05130600929260254}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-15T21:54:44,272 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dd723478fcfc0e25b606965c9135244b 2024-11-15T21:54:44,272 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for dd723478fcfc0e25b606965c9135244b: Running coprocessor pre-open hook at 1731707684235Writing region info on filesystem at 1731707684235Initializing all the Stores at 1731707684243 (+8 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707684243Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707684253 (+10 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707684253Cleaning up temporary data from old regions at 1731707684266 (+13 ms)Running coprocessor post-open hooks at 1731707684272 (+6 ms)Region opened successfully at 1731707684272 2024-11-15T21:54:44,274 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731707682333.dd723478fcfc0e25b606965c9135244b., pid=56, masterSystemTime=1731707683767 2024-11-15T21:54:44,277 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731707682333.dd723478fcfc0e25b606965c9135244b. 2024-11-15T21:54:44,277 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731707682333.dd723478fcfc0e25b606965c9135244b. 2024-11-15T21:54:44,279 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=dd723478fcfc0e25b606965c9135244b, regionState=OPEN, openSeqNum=2, regionLocation=ae3311ea5ccb,32871,1731707679650 2024-11-15T21:54:44,281 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure dd723478fcfc0e25b606965c9135244b, server=ae3311ea5ccb,32871,1731707679650 because future has completed 2024-11-15T21:54:44,285 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=6 2024-11-15T21:54:44,285 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure dd723478fcfc0e25b606965c9135244b, server=ae3311ea5ccb,32871,1731707679650 in 605 msec 2024-11-15T21:54:44,288 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=4 2024-11-15T21:54:44,288 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=dd723478fcfc0e25b606965c9135244b, ASSIGN in 971 msec 2024-11-15T21:54:44,290 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T21:54:44,290 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731707684290"}]},"ts":"1731707684290"} 2024-11-15T21:54:44,293 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, state=ENABLED in hbase:meta 2024-11-15T21:54:44,294 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T21:54:44,299 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true in 1.9500 sec 2024-11-15T21:54:44,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T21:54:44,522 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true completed 2024-11-15T21:54:44,526 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true get assigned. Timeout = 60000ms 2024-11-15T21:54:44,527 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-15T21:54:44,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32871 {}] regionserver.StoreScanner(1138): Switch to stream read (scanned=32802 bytes) of info 2024-11-15T21:54:44,556 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true assigned to meta. Checking AM states. 2024-11-15T21:54:44,557 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-15T21:54:44,558 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true assigned. 2024-11-15T21:54:44,612 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aaa', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731707682333.229a036fd38ae13e7359b15bc2f2e587., hostname=ae3311ea5ccb,40941,1731707679516, seqNum=2] 2024-11-15T21:54:44,614 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aab', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731707682333.229a036fd38ae13e7359b15bc2f2e587., hostname=ae3311ea5ccb,40941,1731707679516, seqNum=2] 2024-11-15T21:54:44,616 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aac', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731707682333.229a036fd38ae13e7359b15bc2f2e587., hostname=ae3311ea5ccb,40941,1731707679516, seqNum=2] 2024-11-15T21:54:44,618 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aae', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731707682333.229a036fd38ae13e7359b15bc2f2e587., hostname=ae3311ea5ccb,40941,1731707679516, seqNum=2] 2024-11-15T21:54:44,619 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aad', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731707682333.229a036fd38ae13e7359b15bc2f2e587., hostname=ae3311ea5ccb,40941,1731707679516, seqNum=2] 2024-11-15T21:54:44,620 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aag', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731707682333.229a036fd38ae13e7359b15bc2f2e587., hostname=ae3311ea5ccb,40941,1731707679516, seqNum=2] 2024-11-15T21:54:44,621 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aaf', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731707682333.229a036fd38ae13e7359b15bc2f2e587., hostname=ae3311ea5ccb,40941,1731707679516, seqNum=2] 2024-11-15T21:54:44,698 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='bbc', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731707682333.dd723478fcfc0e25b606965c9135244b., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:44,700 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='bbd', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731707682333.dd723478fcfc0e25b606965c9135244b., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:44,701 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='bbe', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731707682333.dd723478fcfc0e25b606965c9135244b., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:44,705 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='bbf', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731707682333.dd723478fcfc0e25b606965c9135244b., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:44,706 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='bbb', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731707682333.dd723478fcfc0e25b606965c9135244b., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:44,708 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='bbg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731707682333.dd723478fcfc0e25b606965c9135244b., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:44,709 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='bbh', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731707682333.dd723478fcfc0e25b606965c9135244b., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:44,776 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ccc', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731707682333.50df10b3ae47c5dc7c4521c4c58c192f., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:44,782 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ccd', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731707682333.50df10b3ae47c5dc7c4521c4c58c192f., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:44,784 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='cce', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731707682333.50df10b3ae47c5dc7c4521c4c58c192f., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:44,787 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ccf', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731707682333.50df10b3ae47c5dc7c4521c4c58c192f., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:44,789 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='cch', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731707682333.50df10b3ae47c5dc7c4521c4c58c192f., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:44,793 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ccg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731707682333.50df10b3ae47c5dc7c4521c4c58c192f., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:44,845 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='dde', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731707682333.4256fff2fa33299a6264dd460a4003fb., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:44,848 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddf', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731707682333.4256fff2fa33299a6264dd460a4003fb., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:44,850 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddd', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731707682333.4256fff2fa33299a6264dd460a4003fb., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:44,851 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731707682333.4256fff2fa33299a6264dd460a4003fb., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:44,853 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddh', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731707682333.4256fff2fa33299a6264dd460a4003fb., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:44,860 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddi', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731707682333.4256fff2fa33299a6264dd460a4003fb., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:44,861 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddj', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731707682333.4256fff2fa33299a6264dd460a4003fb., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:44,930 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eee', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731707682333.2c83200b2c422973a68afce1173aae30., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:44,932 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eef', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731707682333.2c83200b2c422973a68afce1173aae30., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:44,937 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eeg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731707682333.2c83200b2c422973a68afce1173aae30., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:44,939 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eeh', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731707682333.2c83200b2c422973a68afce1173aae30., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:44,941 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eei', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731707682333.2c83200b2c422973a68afce1173aae30., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:44,947 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eej', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731707682333.2c83200b2c422973a68afce1173aae30., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:44,949 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eek', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731707682333.2c83200b2c422973a68afce1173aae30., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:45,000 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='fff', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731707682333.3f13d1331edcf64c426c3bf9433330e4., hostname=ae3311ea5ccb,40941,1731707679516, seqNum=2] 2024-11-15T21:54:45,002 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ffg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731707682333.3f13d1331edcf64c426c3bf9433330e4., hostname=ae3311ea5ccb,40941,1731707679516, seqNum=2] 2024-11-15T21:54:45,012 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ffh', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731707682333.3f13d1331edcf64c426c3bf9433330e4., hostname=ae3311ea5ccb,40941,1731707679516, seqNum=2] 2024-11-15T21:54:45,014 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ffi', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731707682333.3f13d1331edcf64c426c3bf9433330e4., hostname=ae3311ea5ccb,40941,1731707679516, seqNum=2] 2024-11-15T21:54:45,018 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ffk', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731707682333.3f13d1331edcf64c426c3bf9433330e4., hostname=ae3311ea5ccb,40941,1731707679516, seqNum=2] 2024-11-15T21:54:45,020 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ffj', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731707682333.3f13d1331edcf64c426c3bf9433330e4., hostname=ae3311ea5ccb,40941,1731707679516, seqNum=2] 2024-11-15T21:54:45,022 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ffl', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731707682333.3f13d1331edcf64c426c3bf9433330e4., hostname=ae3311ea5ccb,40941,1731707679516, seqNum=2] 2024-11-15T21:54:45,071 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggh', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731707682333.f9ef867a298d175eca9d85304f2faa69., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:45,073 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731707682333.f9ef867a298d175eca9d85304f2faa69., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:45,075 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggi', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731707682333.f9ef867a298d175eca9d85304f2faa69., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:45,077 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggj', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731707682333.f9ef867a298d175eca9d85304f2faa69., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:45,083 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggl', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731707682333.f9ef867a298d175eca9d85304f2faa69., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:45,084 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggk', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731707682333.f9ef867a298d175eca9d85304f2faa69., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:45,086 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731707682333.f9ef867a298d175eca9d85304f2faa69., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:45,087 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggn', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731707682333.f9ef867a298d175eca9d85304f2faa69., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:45,109 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhh', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731707682333.0890aeec1f0ca0649e6483a2d56f36b4., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:45,111 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhi', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731707682333.0890aeec1f0ca0649e6483a2d56f36b4., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:45,119 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhj', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731707682333.0890aeec1f0ca0649e6483a2d56f36b4., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:45,121 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhk', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731707682333.0890aeec1f0ca0649e6483a2d56f36b4., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:45,123 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhl', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731707682333.0890aeec1f0ca0649e6483a2d56f36b4., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:45,124 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731707682333.0890aeec1f0ca0649e6483a2d56f36b4., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:45,174 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iii', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731707682333.d28ee8a2ac06d2199c72ac7f884ba470., hostname=ae3311ea5ccb,40941,1731707679516, seqNum=2] 2024-11-15T21:54:45,176 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iij', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731707682333.d28ee8a2ac06d2199c72ac7f884ba470., hostname=ae3311ea5ccb,40941,1731707679516, seqNum=2] 2024-11-15T21:54:45,179 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iik', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731707682333.d28ee8a2ac06d2199c72ac7f884ba470., hostname=ae3311ea5ccb,40941,1731707679516, seqNum=2] 2024-11-15T21:54:45,182 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iil', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731707682333.d28ee8a2ac06d2199c72ac7f884ba470., hostname=ae3311ea5ccb,40941,1731707679516, seqNum=2] 2024-11-15T21:54:45,184 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iim', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731707682333.d28ee8a2ac06d2199c72ac7f884ba470., hostname=ae3311ea5ccb,40941,1731707679516, seqNum=2] 2024-11-15T21:54:45,186 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iin', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731707682333.d28ee8a2ac06d2199c72ac7f884ba470., hostname=ae3311ea5ccb,40941,1731707679516, seqNum=2] 2024-11-15T21:54:45,241 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjj', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731707682333.aec2e46cb64898ba2a66825d3732f6dc., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:45,244 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjk', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731707682333.aec2e46cb64898ba2a66825d3732f6dc., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:45,245 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjl', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731707682333.aec2e46cb64898ba2a66825d3732f6dc., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:45,248 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731707682333.aec2e46cb64898ba2a66825d3732f6dc., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:45,250 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjn', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731707682333.aec2e46cb64898ba2a66825d3732f6dc., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:45,309 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkk', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731707682333.cbb6b5b8341f7ff2ada4ac1931a1a588., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:45,312 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkl', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731707682333.cbb6b5b8341f7ff2ada4ac1931a1a588., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:45,315 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkn', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731707682333.cbb6b5b8341f7ff2ada4ac1931a1a588., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:45,318 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731707682333.cbb6b5b8341f7ff2ada4ac1931a1a588., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:45,320 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731707682333.cbb6b5b8341f7ff2ada4ac1931a1a588., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:45,323 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kko', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731707682333.cbb6b5b8341f7ff2ada4ac1931a1a588., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:45,325 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731707682333.cbb6b5b8341f7ff2ada4ac1931a1a588., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:45,375 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='lll', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731707682333.a3f7be24d811f4d6177c28e08f25af2d., hostname=ae3311ea5ccb,40941,1731707679516, seqNum=2] 2024-11-15T21:54:45,383 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='llm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731707682333.a3f7be24d811f4d6177c28e08f25af2d., hostname=ae3311ea5ccb,40941,1731707679516, seqNum=2] 2024-11-15T21:54:45,387 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='lln', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731707682333.a3f7be24d811f4d6177c28e08f25af2d., hostname=ae3311ea5ccb,40941,1731707679516, seqNum=2] 2024-11-15T21:54:45,402 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='llo', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731707682333.a3f7be24d811f4d6177c28e08f25af2d., hostname=ae3311ea5ccb,40941,1731707679516, seqNum=2] 2024-11-15T21:54:45,405 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='llp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731707682333.a3f7be24d811f4d6177c28e08f25af2d., hostname=ae3311ea5ccb,40941,1731707679516, seqNum=2] 2024-11-15T21:54:45,408 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='llq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731707682333.a3f7be24d811f4d6177c28e08f25af2d., hostname=ae3311ea5ccb,40941,1731707679516, seqNum=2] 2024-11-15T21:54:45,451 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731707682333.6e3b749fcbfdadfe59f1d1cf94f5406a., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:45,455 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmn', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731707682333.6e3b749fcbfdadfe59f1d1cf94f5406a., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:45,457 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmo', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731707682333.6e3b749fcbfdadfe59f1d1cf94f5406a., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:45,459 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731707682333.6e3b749fcbfdadfe59f1d1cf94f5406a., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:45,462 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731707682333.6e3b749fcbfdadfe59f1d1cf94f5406a., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:45,465 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmr', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731707682333.6e3b749fcbfdadfe59f1d1cf94f5406a., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:45,467 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmt', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731707682333.6e3b749fcbfdadfe59f1d1cf94f5406a., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:45,469 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mms', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731707682333.6e3b749fcbfdadfe59f1d1cf94f5406a., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:45,506 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nnn', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731707682333.51af260ba7a9b40a7f788a5c93a416b6., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:45,514 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nno', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731707682333.51af260ba7a9b40a7f788a5c93a416b6., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:45,515 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nnp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731707682333.51af260ba7a9b40a7f788a5c93a416b6., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:45,517 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nnq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731707682333.51af260ba7a9b40a7f788a5c93a416b6., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:45,519 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nnr', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731707682333.51af260ba7a9b40a7f788a5c93a416b6., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:45,520 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nns', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731707682333.51af260ba7a9b40a7f788a5c93a416b6., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:45,530 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nnt', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731707682333.51af260ba7a9b40a7f788a5c93a416b6., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:45,584 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='oop', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731707682333.0ab2d104e772f41ee9c56851adad43f0., hostname=ae3311ea5ccb,40941,1731707679516, seqNum=2] 2024-11-15T21:54:45,587 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ooq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731707682333.0ab2d104e772f41ee9c56851adad43f0., hostname=ae3311ea5ccb,40941,1731707679516, seqNum=2] 2024-11-15T21:54:45,590 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='oor', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731707682333.0ab2d104e772f41ee9c56851adad43f0., hostname=ae3311ea5ccb,40941,1731707679516, seqNum=2] 2024-11-15T21:54:45,591 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='oos', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731707682333.0ab2d104e772f41ee9c56851adad43f0., hostname=ae3311ea5ccb,40941,1731707679516, seqNum=2] 2024-11-15T21:54:45,594 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ooo', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731707682333.0ab2d104e772f41ee9c56851adad43f0., hostname=ae3311ea5ccb,40941,1731707679516, seqNum=2] 2024-11-15T21:54:45,597 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='oot', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731707682333.0ab2d104e772f41ee9c56851adad43f0., hostname=ae3311ea5ccb,40941,1731707679516, seqNum=2] 2024-11-15T21:54:45,599 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='oou', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731707682333.0ab2d104e772f41ee9c56851adad43f0., hostname=ae3311ea5ccb,40941,1731707679516, seqNum=2] 2024-11-15T21:54:45,601 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='oov', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731707682333.0ab2d104e772f41ee9c56851adad43f0., hostname=ae3311ea5ccb,40941,1731707679516, seqNum=2] 2024-11-15T21:54:45,642 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731707682333.3cf264afea186a5cc96edd3e68a10d45., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:45,646 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731707682333.3cf264afea186a5cc96edd3e68a10d45., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:45,651 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppr', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731707682333.3cf264afea186a5cc96edd3e68a10d45., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:45,658 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='pps', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731707682333.3cf264afea186a5cc96edd3e68a10d45., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:45,659 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppt', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731707682333.3cf264afea186a5cc96edd3e68a10d45., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:45,661 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppu', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731707682333.3cf264afea186a5cc96edd3e68a10d45., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:45,663 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731707682333.3cf264afea186a5cc96edd3e68a10d45., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:45,666 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppw', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731707682333.3cf264afea186a5cc96edd3e68a10d45., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:45,693 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731707682333.d100add17d87270d000a3c39b49f8227., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:45,695 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqr', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731707682333.d100add17d87270d000a3c39b49f8227., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:45,697 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqs', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731707682333.d100add17d87270d000a3c39b49f8227., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:45,699 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqt', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731707682333.d100add17d87270d000a3c39b49f8227., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:45,701 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqu', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731707682333.d100add17d87270d000a3c39b49f8227., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:45,703 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731707682333.d100add17d87270d000a3c39b49f8227., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:45,736 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rrr', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731707682333.d9989b0341ef96818e811f874dbcf91c., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:45,738 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rrs', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731707682333.d9989b0341ef96818e811f874dbcf91c., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:45,740 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rrt', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731707682333.d9989b0341ef96818e811f874dbcf91c., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:45,742 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rru', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731707682333.d9989b0341ef96818e811f874dbcf91c., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:45,744 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rrv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731707682333.d9989b0341ef96818e811f874dbcf91c., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:45,791 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='sss', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731707682333.386f4af1e738f142f3fbde37773d4d2c., hostname=ae3311ea5ccb,40941,1731707679516, seqNum=2] 2024-11-15T21:54:45,793 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='sst', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731707682333.386f4af1e738f142f3fbde37773d4d2c., hostname=ae3311ea5ccb,40941,1731707679516, seqNum=2] 2024-11-15T21:54:45,795 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ssu', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731707682333.386f4af1e738f142f3fbde37773d4d2c., hostname=ae3311ea5ccb,40941,1731707679516, seqNum=2] 2024-11-15T21:54:45,797 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ssv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731707682333.386f4af1e738f142f3fbde37773d4d2c., hostname=ae3311ea5ccb,40941,1731707679516, seqNum=2] 2024-11-15T21:54:45,799 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ssw', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731707682333.386f4af1e738f142f3fbde37773d4d2c., hostname=ae3311ea5ccb,40941,1731707679516, seqNum=2] 2024-11-15T21:54:45,801 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ssx', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731707682333.386f4af1e738f142f3fbde37773d4d2c., hostname=ae3311ea5ccb,40941,1731707679516, seqNum=2] 2024-11-15T21:54:45,840 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ttt', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731707682333.d6063c53955b74f03cc36bd578a6bd41., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:45,841 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ttu', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731707682333.d6063c53955b74f03cc36bd578a6bd41., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:45,843 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ttw', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731707682333.d6063c53955b74f03cc36bd578a6bd41., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:45,845 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ttv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731707682333.d6063c53955b74f03cc36bd578a6bd41., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:45,846 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ttx', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731707682333.d6063c53955b74f03cc36bd578a6bd41., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:45,848 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='tty', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731707682333.d6063c53955b74f03cc36bd578a6bd41., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:45,850 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ttz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731707682333.d6063c53955b74f03cc36bd578a6bd41., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:45,897 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uuu', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731707682333.fac6af87aa0fc9826df0515f766c031e., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:45,899 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uuv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731707682333.fac6af87aa0fc9826df0515f766c031e., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:45,901 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uuw', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731707682333.fac6af87aa0fc9826df0515f766c031e., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:45,903 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uux', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731707682333.fac6af87aa0fc9826df0515f766c031e., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:45,905 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uuy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731707682333.fac6af87aa0fc9826df0515f766c031e., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:45,907 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uuz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731707682333.fac6af87aa0fc9826df0515f766c031e., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:45,959 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvw', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731707682333.5ce283f02f0974e39c98759bb2bc9883., hostname=ae3311ea5ccb,40941,1731707679516, seqNum=2] 2024-11-15T21:54:45,962 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731707682333.5ce283f02f0974e39c98759bb2bc9883., hostname=ae3311ea5ccb,40941,1731707679516, seqNum=2] 2024-11-15T21:54:45,971 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvx', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731707682333.5ce283f02f0974e39c98759bb2bc9883., hostname=ae3311ea5ccb,40941,1731707679516, seqNum=2] 2024-11-15T21:54:45,973 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731707682333.5ce283f02f0974e39c98759bb2bc9883., hostname=ae3311ea5ccb,40941,1731707679516, seqNum=2] 2024-11-15T21:54:45,976 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731707682333.5ce283f02f0974e39c98759bb2bc9883., hostname=ae3311ea5ccb,40941,1731707679516, seqNum=2] 2024-11-15T21:54:45,977 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vwa', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731707682333.5ce283f02f0974e39c98759bb2bc9883., hostname=ae3311ea5ccb,40941,1731707679516, seqNum=2] 2024-11-15T21:54:45,979 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vwb', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731707682333.5ce283f02f0974e39c98759bb2bc9883., hostname=ae3311ea5ccb,40941,1731707679516, seqNum=2] 2024-11-15T21:54:45,981 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vwc', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731707682333.5ce283f02f0974e39c98759bb2bc9883., hostname=ae3311ea5ccb,40941,1731707679516, seqNum=2] 2024-11-15T21:54:46,013 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='www', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731707682333.6694325f9c862a5a1ee02eabf7861cc3., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:46,015 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='wwx', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731707682333.6694325f9c862a5a1ee02eabf7861cc3., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:46,017 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='wwy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731707682333.6694325f9c862a5a1ee02eabf7861cc3., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:46,018 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='wwz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731707682333.6694325f9c862a5a1ee02eabf7861cc3., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:46,020 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='wxa', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731707682333.6694325f9c862a5a1ee02eabf7861cc3., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2] 2024-11-15T21:54:46,063 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xxx', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731707682333.46b5f83abe9585829b26884a1b26b43f., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:46,066 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xxy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731707682333.46b5f83abe9585829b26884a1b26b43f., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:46,068 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xxz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731707682333.46b5f83abe9585829b26884a1b26b43f., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:46,071 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xyb', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731707682333.46b5f83abe9585829b26884a1b26b43f., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:46,073 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xya', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731707682333.46b5f83abe9585829b26884a1b26b43f., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:46,121 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yyy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731707682333.1031049c0b4149d1189db33e38ab56cd., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:46,124 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yyz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731707682333.1031049c0b4149d1189db33e38ab56cd., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:46,132 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yza', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731707682333.1031049c0b4149d1189db33e38ab56cd., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:46,134 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yzb', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731707682333.1031049c0b4149d1189db33e38ab56cd., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:46,136 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yzc', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731707682333.1031049c0b4149d1189db33e38ab56cd., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=2] 2024-11-15T21:54:46,182 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='zzz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731707682333.e76ae188febdca91d319999f619d9e11., hostname=ae3311ea5ccb,40941,1731707679516, seqNum=2] 2024-11-15T21:54:46,250 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T21:54:46,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32871 {}] regionserver.RSRpcServices(2610): Large batch operation detected (greater than 5000) (HBASE-18023). Requested Number of Rows: 6327 Client: jenkins//172.17.0.3 first region in multi=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731707682333.dd723478fcfc0e25b606965c9135244b. 2024-11-15T21:54:46,269 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53724, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T21:54:46,270 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T21:54:46,314 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43696, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T21:54:46,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37455 {}] regionserver.RSRpcServices(2610): Large batch operation detected (greater than 5000) (HBASE-18023). Requested Number of Rows: 6327 Client: jenkins//172.17.0.3 first region in multi=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731707682333.50df10b3ae47c5dc7c4521c4c58c192f. 2024-11-15T21:54:47,362 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T21:54:47,476 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-15T21:54:47,478 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true' 2024-11-15T21:54:47,485 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,, stopping at row=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true ,, for max=2147483647 with caching=100 2024-11-15T21:54:47,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32871 {}] regionserver.StoreScanner(1138): Switch to stream read (scanned=32802 bytes) of info 2024-11-15T21:54:47,745 WARN [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: true 2024-11-15T21:54:47,745 INFO [Time-limited test {}] assignment.AssignmentTestingUtil(82): KILL REGION SERVER ae3311ea5ccb,32871,1731707679650 2024-11-15T21:54:47,745 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing ae3311ea5ccb,32871,1731707679650 2024-11-15T21:54:47,746 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server ae3311ea5ccb,32871,1731707679650: Simulated kill ***** 2024-11-15T21:54:47,746 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-15T21:54:47,749 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-15T21:54:47,751 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-15T21:54:47,755 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-15T21:54:47,756 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-15T21:54:47,762 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 633215392 }, "NonHeapMemoryUsage": { "committed": 157417472, "init": 7667712, "max": -1, "used": 154798856 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "ae3311ea5ccb", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 2, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 1517960, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 382, "ProcessCallTime_min": 0, "ProcessCallTime_max": 683, "ProcessCallTime_mean": 6, "ProcessCallTime_25th_percentile": 170, "ProcessCallTime_median": 341, "ProcessCallTime_75th_percentile": 512, "ProcessCallTime_90th_percentile": 614, "ProcessCallTime_95th_percentile": 648, "ProcessCallTime_98th_percentile": 669, "ProcessCallTime_99th_percentile": 676, "ProcessCallTime_99.9th_percentile": 682, "ProcessCallTime_TimeRangeCount_0-1": 382, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 382, "QueueCallTime_min": 0, "QueueCallTime_max": 114, "QueueCallTime_mean": 2, "QueueCallTime_25th_percentile": 28, "QueueCallTime_median": 57, "QueueCallTime_75th_percentile": 85, "QueueCallTime_90th_percentile": 102, "QueueCallTime_95th_percentile": 108, "QueueCallTime_98th_percentile": 111, "QueueCallTime_99th_percentile": 112, "QueueCallTime_99.9th_percentile": 113, "QueueCallTime_TimeRangeCount_0-1": 382, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 382, "TotalCallTime_min": 0, "TotalCallTime_max": 727, "TotalCallTime_mean": 8, "TotalCallTime_25th_percentile": 181, "TotalCallTime_median": 363, "TotalCallTime_75th_percentile": 545, "TotalCallTime_90th_percentile": 654, "TotalCallTime_95th_percentile": 690, "TotalCallTime_98th_percentile": 712, "TotalCallTime_99th_percentile": 719, "TotalCallTime_99.9th_percentile": 726, "TotalCallTime_TimeRangeCount_0-1": 382, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 382, "ResponseSize_min": 0, "ResponseSize_max": 59075, "ResponseSize_mean": 12464, "ResponseSize_25th_percentile": 14768, "ResponseSize_median": 29537, "ResponseSize_75th_percentile": 44306, "ResponseSize_90th_percentile": 53167, "ResponseSize_95th_percentile": 56121, "ResponseSize_98th_percentile": 57893, "ResponseSize_99th_percentile": 58484, "ResponseSize_99.9th_percentile": 59015, "ResponseSize_SizeRangeCount_0-10": 382, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 382, "RequestSize_min": 41, "RequestSize_max": 508560, "RequestSize_mean": 3961, "RequestSize_25th_percentile": 127170, "RequestSize_median": 254300, "RequestSize_75th_percentile": 381430, "RequestSize_90th_percentile": 457708, "RequestSize_95th_percentile": 483134, "RequestSize_98th_percentile": 498389, "RequestSize_99th_percentile": 503474, "RequestSize_99.9th_percentile": 508051, "RequestSize_SizeRangeCount_0-10": 382, "sentBytes": 3559677 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "ae3311ea5ccb", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:58023", "tag.serverName": "ae3311ea5ccb,40941,1731707679516", "tag.clusterId": "4680bf58-6f3b-4f68-8171-8cc0087dc31c", "tag.Context": "regionserver", "tag.Hostname": "ae3311ea5ccb", "regionCount": 8, "storeCount": 24, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 6144, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1731707679516, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 0, "localBytesRead": 0, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 0, "ByteBuffAllocatorPoolAllocationBytes": 998400, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 16, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 15, "rpcScanRequestCount": 15, "rpcMultiRequestCount": 1, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 15, "ScanTime_min": 0, "ScanTime_max": 7, "ScanTime_mean": 2, "ScanTime_25th_percentile": 1, "ScanTime_median": 3, "ScanTime_75th_percentile": 5, "ScanTime_90th_percentile": 6, "ScanTime_95th_percentile": 6, "ScanTime_98th_percentile": 6, "ScanTime_99th_percentile": 6, "ScanTime_99.9th_percentile": 6, "ScanTime_TimeRangeCount_0-1": 15, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 8, "PutBatch_min": 3, "PutBatch_max": 107, "PutBatch_mean": 51, "PutBatch_25th_percentile": 29, "PutBatch_median": 55, "PutBatch_75th_percentile": 81, "PutBatch_90th_percentile": 96, "PutBatch_95th_percentile": 101, "PutBatch_98th_percentile": 104, "PutBatch_99th_percentile": 105, "PutBatch_99.9th_percentile": 106, "PutBatch_TimeRangeCount_0-1": 8, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 15, "ScanSize_min": 80, "ScanSize_max": 56240, "ScanSize_mean": 52496, "ScanSize_25th_percentile": 14120, "ScanSize_median": 28160, "ScanSize_75th_percentile": 42200, "ScanSize_90th_percentile": 50624, "ScanSize_95th_percentile": 53432, "ScanSize_98th_percentile": 55116, "ScanSize_99th_percentile": 55678, "ScanSize_99.9th_percentile": 56183, "ScanSize_SizeRangeCount_0-10": 15, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-15T21:54:47,770 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36531 {}] master.MasterRpcServices(700): ae3311ea5ccb,32871,1731707679650 reported a fatal error: ***** ABORTING region server ae3311ea5ccb,32871,1731707679650: Simulated kill ***** 2024-11-15T21:54:47,786 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ae3311ea5ccb,32871,1731707679650' ***** 2024-11-15T21:54:47,786 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-15T21:54:47,787 INFO [RS:1;ae3311ea5ccb:32871 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-15T21:54:47,787 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-15T21:54:47,787 INFO [RS:1;ae3311ea5ccb:32871 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-15T21:54:47,787 INFO [RS:1;ae3311ea5ccb:32871 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-15T21:54:47,787 INFO [RS:1;ae3311ea5ccb:32871 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-15T21:54:47,788 INFO [RS:1;ae3311ea5ccb:32871 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;ae3311ea5ccb:32871. 2024-11-15T21:54:47,788 DEBUG [RS:1;ae3311ea5ccb:32871 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T21:54:47,788 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-15T21:54:47,794 DEBUG [RS:1;ae3311ea5ccb:32871 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T21:54:47,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741836_1012 (size=54214) 2024-11-15T21:54:47,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741836_1012 (size=54214) 2024-11-15T21:54:47,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741836_1012 (size=54214) 2024-11-15T21:54:47,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741835_1011 (size=334920) 2024-11-15T21:54:47,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741835_1011 (size=334920) 2024-11-15T21:54:47,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741835_1011 (size=334920) 2024-11-15T21:54:47,824 DEBUG [RS:1;ae3311ea5ccb:32871 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T21:54:47,824 INFO [RS:1;ae3311ea5ccb:32871 {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T21:54:47,825 INFO [RS:1;ae3311ea5ccb:32871 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:32871 2024-11-15T21:54:47,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36531-0x100f86521fe0000, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T21:54:47,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32871-0x100f86521fe0002, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ae3311ea5ccb,32871,1731707679650 2024-11-15T21:54:47,836 INFO [RS:1;ae3311ea5ccb:32871 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-15T21:54:47,839 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ae3311ea5ccb,32871,1731707679650] 2024-11-15T21:54:47,841 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ae3311ea5ccb,32871,1731707679650 already deleted, retry=false 2024-11-15T21:54:47,842 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of ae3311ea5ccb,32871,1731707679650 on ae3311ea5ccb,36531,1731707678636 2024-11-15T21:54:47,845 INFO [regionserver/ae3311ea5ccb:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T21:54:47,850 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure ae3311ea5ccb,32871,1731707679650, splitWal=true, meta=true 2024-11-15T21:54:47,855 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=57 for ae3311ea5ccb,32871,1731707679650 (carryingMeta=true) ae3311ea5ccb,32871,1731707679650/CRASHED/regionCount=10/lock=java.util.concurrent.locks.ReentrantReadWriteLock@69113e70[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-15T21:54:47,858 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure ae3311ea5ccb,32871,1731707679650, splitWal=true, meta=true 2024-11-15T21:54:47,858 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-15T21:54:47,858 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure ae3311ea5ccb,32871,1731707679650, splitWal=true, meta=true 2024-11-15T21:54:47,859 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T21:54:47,859 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure ae3311ea5ccb,32871,1731707679650, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T21:54:47,888 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-15T21:54:47,890 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 0 exec state=pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure ae3311ea5ccb,32871,1731707679650, splitWal=true, meta=true 2024-11-15T21:54:47,891 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T21:54:47,891 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T21:54:47,891 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T21:54:47,892 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T21:54:47,892 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T21:54:47,893 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T21:54:47,893 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T21:54:47,893 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T21:54:47,895 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-15T21:54:47,902 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T21:54:47,903 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T21:54:47,904 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-15T21:54:47,904 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure ae3311ea5ccb,32871,1731707679650, splitWal=true, meta=true 2024-11-15T21:54:47,904 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure ae3311ea5ccb,32871,1731707679650, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T21:54:47,905 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-15T21:54:47,910 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ae3311ea5ccb,32871,1731707679650, regionLocation=ae3311ea5ccb,32871,1731707679650, openSeqNum=2 2024-11-15T21:54:47,910 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ae3311ea5ccb,32871,1731707679650, table=hbase:meta, region=1588230740 2024-11-15T21:54:47,910 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ae3311ea5ccb,32871,1731707679650, state=OPEN 2024-11-15T21:54:47,912 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36531-0x100f86521fe0000, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T21:54:47,913 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40941-0x100f86521fe0001, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T21:54:47,913 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37455-0x100f86521fe0003, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T21:54:47,913 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T21:54:47,913 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T21:54:47,913 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T21:54:47,919 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731707717919 2024-11-15T21:54:47,919 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-15T21:54:47,920 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure ae3311ea5ccb,32871,1731707679650, splitWal=true, meta=true 2024-11-15T21:54:47,920 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-15T21:54:47,921 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server ae3311ea5ccb:32871 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T21:54:47,923 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1, error=java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:54:47,924 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-15T21:54:47,924 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 from cache 2024-11-15T21:54:47,926 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address ae3311ea5ccb:32871 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:54:47,926 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure ae3311ea5ccb,32871,1731707679650, splitWal=true, meta=true, isMeta: true 2024-11-15T21:54:47,929 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/WALs/ae3311ea5ccb,32871,1731707679650-splitting 2024-11-15T21:54:47,931 INFO [PEWorker-1 {}] master.SplitWALManager(105): ae3311ea5ccb,32871,1731707679650 WAL count=1, meta=true 2024-11-15T21:54:47,933 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure ae3311ea5ccb%2C32871%2C1731707679650.meta.1731707681806.meta}] 2024-11-15T21:54:47,933 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-15T21:54:47,933 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, hasLock=true; ServerCrashProcedure ae3311ea5ccb,32871,1731707679650, splitWal=true, meta=true 2024-11-15T21:54:47,933 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T21:54:47,933 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, hasLock=true; ServerCrashProcedure ae3311ea5ccb,32871,1731707679650, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T21:54:47,939 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32871-0x100f86521fe0002, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T21:54:47,939 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32871-0x100f86521fe0002, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T21:54:47,940 INFO [RS:1;ae3311ea5ccb:32871 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-15T21:54:47,940 INFO [RS:1;ae3311ea5ccb:32871 {}] regionserver.HRegionServer(1031): Exiting; stopping=ae3311ea5ccb,32871,1731707679650; zookeeper connection closed. 2024-11-15T21:54:47,941 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@387ca9b0 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@387ca9b0 2024-11-15T21:54:48,035 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T21:54:48,035 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1] 2024-11-15T21:54:48,036 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to ae3311ea5ccb:32871 this server is in the failed servers list 2024-11-15T21:54:48,036 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=ae3311ea5ccb:32871 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:54:48,036 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:54:48,036 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 from cache 2024-11-15T21:54:48,244 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T21:54:48,244 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1] 2024-11-15T21:54:48,245 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to ae3311ea5ccb:32871 this server is in the failed servers list 2024-11-15T21:54:48,245 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=ae3311ea5ccb:32871 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:54:48,245 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:54:48,245 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 from cache 2024-11-15T21:54:48,282 INFO [regionserver/ae3311ea5ccb:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-15T21:54:48,282 INFO [regionserver/ae3311ea5ccb:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-15T21:54:48,555 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T21:54:48,555 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1] 2024-11-15T21:54:48,555 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to ae3311ea5ccb:32871 this server is in the failed servers list 2024-11-15T21:54:48,556 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=ae3311ea5ccb:32871 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:54:48,556 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:54:48,556 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 from cache 2024-11-15T21:54:49,065 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T21:54:49,065 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1] 2024-11-15T21:54:49,065 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to ae3311ea5ccb:32871 this server is in the failed servers list 2024-11-15T21:54:49,066 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=ae3311ea5ccb:32871 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:54:49,066 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:54:49,066 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 from cache 2024-11-15T21:54:49,131 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-15T21:54:49,132 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-15T21:54:49,134 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-15T21:54:49,134 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true Metrics about Tables on a single HBase RegionServer 2024-11-15T21:54:49,135 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-15T21:54:49,135 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-15T21:54:49,135 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-15T21:54:49,136 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-15T21:54:50,075 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T21:54:50,075 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1] 2024-11-15T21:54:50,076 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server ae3311ea5ccb:32871 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T21:54:50,080 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,32871,1731707679650 for '' in 1588230740 of hbase:meta failed, tries = 6, maxAttempts = 46, timeout = 60000 ms, time elapsed = 2157 ms java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-15T21:54:50,081 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1, error=java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:54:50,081 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-15T21:54:50,081 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 from cache 2024-11-15T21:54:50,081 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address ae3311ea5ccb:32871 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:54:52,105 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T21:54:52,105 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1] 2024-11-15T21:54:52,106 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server ae3311ea5ccb:32871 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T21:54:52,107 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,32871,1731707679650 for '' in 1588230740 of hbase:meta failed, tries = 7, maxAttempts = 46, timeout = 60000 ms, time elapsed = 4187 ms java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-15T21:54:52,108 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1, error=java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:54:52,108 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-15T21:54:52,108 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 from cache 2024-11-15T21:54:52,108 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address ae3311ea5ccb:32871 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:54:56,114 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T21:54:56,114 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1] 2024-11-15T21:54:56,116 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server ae3311ea5ccb:32871 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T21:54:56,116 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,32871,1731707679650 for '' in 1588230740 of hbase:meta failed, tries = 8, maxAttempts = 46, timeout = 60000 ms, time elapsed = 8196 ms java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-15T21:54:56,117 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1, error=java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:54:56,117 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-15T21:54:56,117 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 from cache 2024-11-15T21:54:56,117 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address ae3311ea5ccb:32871 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:55:06,185 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T21:55:06,185 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1] 2024-11-15T21:55:06,186 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server ae3311ea5ccb:32871 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T21:55:06,186 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,32871,1731707679650 for '' in 1588230740 of hbase:meta failed, tries = 9, maxAttempts = 46, timeout = 60000 ms, time elapsed = 18266 ms java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-15T21:55:06,187 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1, error=java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:55:06,187 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-15T21:55:06,187 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 from cache 2024-11-15T21:55:06,187 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address ae3311ea5ccb:32871 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:55:07,960 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T21:55:14,800 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=ae3311ea5ccb:36531 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] ... 10 more 2024-11-15T21:55:16,205 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T21:55:16,205 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1] 2024-11-15T21:55:16,206 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server ae3311ea5ccb:32871 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T21:55:16,206 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,32871,1731707679650 for '' in 1588230740 of hbase:meta failed, tries = 10, maxAttempts = 46, timeout = 60000 ms, time elapsed = 28286 ms java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-15T21:55:16,207 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1, error=java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:55:16,207 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-15T21:55:16,207 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 from cache 2024-11-15T21:55:16,207 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address ae3311ea5ccb:32871 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:55:22,886 INFO [master/ae3311ea5ccb:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-15T21:55:22,886 INFO [master/ae3311ea5ccb:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-15T21:55:26,234 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T21:55:26,234 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1] 2024-11-15T21:55:26,235 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server ae3311ea5ccb:32871 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T21:55:26,236 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,32871,1731707679650 for '' in 1588230740 of hbase:meta failed, tries = 11, maxAttempts = 46, timeout = 60000 ms, time elapsed = 38316 ms java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-15T21:55:26,236 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1, error=java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:55:26,236 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-15T21:55:26,236 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 from cache 2024-11-15T21:55:26,237 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address ae3311ea5ccb:32871 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:55:30,642 WARN [RS:0;ae3311ea5ccb:40941 {}] monitoring.TaskMonitor(166): Status Processing ServerCrashProcedure of ae3311ea5ccb,32871,1731707679650: status=status unset, state=RUNNING, startTime=1731707687858, completionTime=-1 appears to have been leaked 2024-11-15T21:55:36,255 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T21:55:36,255 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1] 2024-11-15T21:55:36,256 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server ae3311ea5ccb:32871 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T21:55:36,257 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,32871,1731707679650 for '' in 1588230740 of hbase:meta failed, tries = 12, maxAttempts = 46, timeout = 60000 ms, time elapsed = 48336 ms java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-15T21:55:36,257 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1, error=java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:55:36,257 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-15T21:55:36,257 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 from cache 2024-11-15T21:55:36,257 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address ae3311ea5ccb:32871 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:55:37,961 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T21:55:41,207 INFO [regionserver/ae3311ea5ccb:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: ExecutorStatusChore was stopped 2024-11-15T21:55:42,224 DEBUG [master/ae3311ea5ccb:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 50df10b3ae47c5dc7c4521c4c58c192f changed from -1.0 to 0.0, refreshing cache 2024-11-15T21:55:42,225 DEBUG [master/ae3311ea5ccb:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 2c83200b2c422973a68afce1173aae30 changed from -1.0 to 0.0, refreshing cache 2024-11-15T21:55:42,225 DEBUG [master/ae3311ea5ccb:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 0890aeec1f0ca0649e6483a2d56f36b4 changed from -1.0 to 0.0, refreshing cache 2024-11-15T21:55:42,226 DEBUG [master/ae3311ea5ccb:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region cbb6b5b8341f7ff2ada4ac1931a1a588 changed from -1.0 to 0.0, refreshing cache 2024-11-15T21:55:42,226 DEBUG [master/ae3311ea5ccb:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 51af260ba7a9b40a7f788a5c93a416b6 changed from -1.0 to 0.0, refreshing cache 2024-11-15T21:55:42,226 DEBUG [master/ae3311ea5ccb:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region d100add17d87270d000a3c39b49f8227 changed from -1.0 to 0.0, refreshing cache 2024-11-15T21:55:42,226 DEBUG [master/ae3311ea5ccb:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region fac6af87aa0fc9826df0515f766c031e changed from -1.0 to 0.0, refreshing cache 2024-11-15T21:55:42,226 DEBUG [master/ae3311ea5ccb:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 46b5f83abe9585829b26884a1b26b43f changed from -1.0 to 0.0, refreshing cache 2024-11-15T21:55:42,226 DEBUG [master/ae3311ea5ccb:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1031049c0b4149d1189db33e38ab56cd changed from -1.0 to 0.0, refreshing cache 2024-11-15T21:55:42,226 DEBUG [master/ae3311ea5ccb:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 229a036fd38ae13e7359b15bc2f2e587 changed from -1.0 to 0.0, refreshing cache 2024-11-15T21:55:42,226 DEBUG [master/ae3311ea5ccb:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 3f13d1331edcf64c426c3bf9433330e4 changed from -1.0 to 0.0, refreshing cache 2024-11-15T21:55:42,227 DEBUG [master/ae3311ea5ccb:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region d28ee8a2ac06d2199c72ac7f884ba470 changed from -1.0 to 0.0, refreshing cache 2024-11-15T21:55:42,227 DEBUG [master/ae3311ea5ccb:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region a3f7be24d811f4d6177c28e08f25af2d changed from -1.0 to 0.0, refreshing cache 2024-11-15T21:55:42,227 DEBUG [master/ae3311ea5ccb:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 0ab2d104e772f41ee9c56851adad43f0 changed from -1.0 to 0.0, refreshing cache 2024-11-15T21:55:42,227 DEBUG [master/ae3311ea5ccb:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 386f4af1e738f142f3fbde37773d4d2c changed from -1.0 to 0.0, refreshing cache 2024-11-15T21:55:42,227 DEBUG [master/ae3311ea5ccb:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5ce283f02f0974e39c98759bb2bc9883 changed from -1.0 to 0.0, refreshing cache 2024-11-15T21:55:42,227 DEBUG [master/ae3311ea5ccb:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e76ae188febdca91d319999f619d9e11 changed from -1.0 to 0.0, refreshing cache 2024-11-15T21:55:47,802 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=ae3311ea5ccb:36531 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] ... 10 more 2024-11-15T21:55:47,925 WARN [Time-limited test {}] procedure.MasterProcedureTestingUtility$3(130): Failed to load meta org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=13, exceptions: 2024-11-15T21:54:47.925Z, java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:54:48.036Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=ae3311ea5ccb:32871 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:54:48.245Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=ae3311ea5ccb:32871 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:54:48.556Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=ae3311ea5ccb:32871 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:54:49.066Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=ae3311ea5ccb:32871 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:54:50.081Z, java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:54:52.108Z, java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:54:56.117Z, java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:55:06.187Z, java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:55:16.207Z, java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:55:26.236Z, java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:55:36.257Z, java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:493) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:461) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:422) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.fullScanRegions(MetaTableAccessor.java:108) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionStateStore.visitMeta(RegionStateStore.java:108) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.loadMeta(AssignmentManager.java:1907) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.joinCluster(AssignmentManager.java:1769) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:126) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:121) ~[test-classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:143) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:477) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:486) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPBase.testRecoveryAndDoubleExecution(TestSCPBase.java:114) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPWithMeta.testRecoveryAndDoubleExecutionOnRsWithMeta(TestSCPWithMeta.java:36) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:102) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-15T21:55:47,926 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 1 exec state=pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, hasLock=true; ServerCrashProcedure ae3311ea5ccb,32871,1731707679650, splitWal=true, meta=true 2024-11-15T21:55:47,926 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T21:55:47,926 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T21:55:47,926 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T21:55:47,927 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T21:55:47,927 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T21:55:47,927 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T21:55:47,927 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T21:55:47,927 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T21:55:47,928 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T21:55:47,930 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T21:55:47,931 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T21:55:47,932 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-15T21:55:47,932 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=false; ServerCrashProcedure ae3311ea5ccb,32871,1731707679650, splitWal=true, meta=true 2024-11-15T21:55:47,932 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=false; ServerCrashProcedure ae3311ea5ccb,32871,1731707679650, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T21:55:47,932 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-15T21:55:47,933 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ae3311ea5ccb,32871,1731707679650, regionLocation=ae3311ea5ccb,32871,1731707679650, openSeqNum=2 2024-11-15T21:55:47,933 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ae3311ea5ccb,32871,1731707679650, table=hbase:meta, region=1588230740 2024-11-15T21:55:47,934 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ae3311ea5ccb,32871,1731707679650, state=OPEN 2024-11-15T21:55:47,936 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37455-0x100f86521fe0003, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T21:55:47,936 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40941-0x100f86521fe0001, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T21:55:47,936 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T21:55:47,936 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T21:55:47,936 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36531-0x100f86521fe0000, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T21:55:47,936 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T21:55:47,941 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731707777941 2024-11-15T21:55:47,941 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-15T21:55:47,941 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure ae3311ea5ccb,32871,1731707679650, splitWal=true, meta=true, isMeta: true 2024-11-15T21:55:47,942 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T21:55:47,942 DEBUG [Time-limited test {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1] 2024-11-15T21:55:47,943 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server ae3311ea5ccb:32871 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T21:55:47,944 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address ae3311ea5ccb:32871 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:55:47,944 DEBUG [Time-limited test {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1, error=java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:55:47,944 DEBUG [Time-limited test {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-15T21:55:47,944 DEBUG [Time-limited test {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 from cache 2024-11-15T21:55:47,946 INFO [PEWorker-1 {}] master.SplitWALManager(105): ae3311ea5ccb,32871,1731707679650 WAL count=1, meta=true 2024-11-15T21:55:47,946 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure ae3311ea5ccb%2C32871%2C1731707679650.meta.1731707681806.meta}] 2024-11-15T21:55:47,946 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-15T21:55:47,953 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:55:47,955 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure ae3311ea5ccb%2C32871%2C1731707679650.meta.1731707681806.meta, worker=ae3311ea5ccb,37455,1731707679725}] 2024-11-15T21:55:48,055 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T21:55:48,055 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1] 2024-11-15T21:55:48,055 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to ae3311ea5ccb:32871 this server is in the failed servers list 2024-11-15T21:55:48,056 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=ae3311ea5ccb:32871 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:55:48,056 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:55:48,056 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 from cache 2024-11-15T21:55:48,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=59 2024-11-15T21:55:48,152 INFO [RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(299): Splitting hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/WALs/ae3311ea5ccb,32871,1731707679650-splitting/ae3311ea5ccb%2C32871%2C1731707679650.meta.1731707681806.meta, size=52.9 K (54214bytes) 2024-11-15T21:55:48,153 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-15T21:55:48,154 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-15T21:55:48,154 INFO [RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/WALs/ae3311ea5ccb,32871,1731707679650-splitting/ae3311ea5ccb%2C32871%2C1731707679650.meta.1731707681806.meta 2024-11-15T21:55:48,163 INFO [RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/WALs/ae3311ea5ccb,32871,1731707679650-splitting/ae3311ea5ccb%2C32871%2C1731707679650.meta.1731707681806.meta after 6ms 2024-11-15T21:55:48,177 INFO [RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(310): Open hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/WALs/ae3311ea5ccb,32871,1731707679650-splitting/ae3311ea5ccb%2C32871%2C1731707679650.meta.1731707681806.meta took 27ms 2024-11-15T21:55:48,193 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 2024-11-15T21:55:48,202 INFO [RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.meta.1731707681806.meta.temp 2024-11-15T21:55:48,203 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/WALs/ae3311ea5ccb,32871,1731707679650-splitting/ae3311ea5ccb%2C32871%2C1731707679650.meta.1731707681806.meta so closing down 2024-11-15T21:55:48,203 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-15T21:55:48,206 INFO [RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.meta.1731707681806.meta.temp 2024-11-15T21:55:48,209 INFO [RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(145): 3 split writer threads finished 2024-11-15T21:55:48,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741864_1040 (size=53973) 2024-11-15T21:55:48,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741864_1040 (size=53973) 2024-11-15T21:55:48,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741864_1040 (size=53973) 2024-11-15T21:55:48,223 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.meta.1731707681806.meta.temp (wrote 57 edits, skipped 0 edits in 2 ms) 2024-11-15T21:55:48,226 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.meta.1731707681806.meta.temp to hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-15T21:55:48,230 INFO [RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(425): Processed 58 edits across 1 Regions in 42 ms; skipped=1; WAL=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/WALs/ae3311ea5ccb,32871,1731707679650-splitting/ae3311ea5ccb%2C32871%2C1731707679650.meta.1731707681806.meta, size=52.9 K, length=54214, corrupted=false, cancelled=false 2024-11-15T21:55:48,231 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(428): Completed split of hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/WALs/ae3311ea5ccb,32871,1731707679650-splitting/ae3311ea5ccb%2C32871%2C1731707679650.meta.1731707681806.meta, journal: Splitting hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/WALs/ae3311ea5ccb,32871,1731707679650-splitting/ae3311ea5ccb%2C32871%2C1731707679650.meta.1731707681806.meta, size=52.9 K (54214bytes) at 1731707748152Finishing writing output for hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/WALs/ae3311ea5ccb,32871,1731707679650-splitting/ae3311ea5ccb%2C32871%2C1731707679650.meta.1731707681806.meta so closing down at 1731707748203 (+51 ms)Creating recovered edits writer path=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.meta.1731707681806.meta.temp at 1731707748206 (+3 ms)3 split writer threads finished at 1731707748209 (+3 ms)Closed recovered edits writer path=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.meta.1731707681806.meta.temp (wrote 57 edits, skipped 0 edits in 2 ms) at 1731707748224 (+15 ms)Rename recovered edits hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.meta.1731707681806.meta.temp to hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 at 1731707748226 (+2 ms)Processed 58 edits across 1 Regions in 42 ms; skipped=1; WAL=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/WALs/ae3311ea5ccb,32871,1731707679650-splitting/ae3311ea5ccb%2C32871%2C1731707679650.meta.1731707681806.meta, size=52.9 K, length=54214, corrupted=false, cancelled=false at 1731707748231 (+5 ms) 2024-11-15T21:55:48,231 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/WALs/ae3311ea5ccb,32871,1731707679650-splitting/ae3311ea5ccb%2C32871%2C1731707679650.meta.1731707681806.meta 2024-11-15T21:55:48,233 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-15T21:55:48,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36531 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-15T21:55:48,243 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/WALs/ae3311ea5ccb,32871,1731707679650-splitting/ae3311ea5ccb%2C32871%2C1731707679650.meta.1731707681806.meta to hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/oldWALs 2024-11-15T21:55:48,246 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-15T21:55:48,246 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ae3311ea5ccb%2C32871%2C1731707679650.meta.1731707681806.meta, worker=ae3311ea5ccb,37455,1731707679725 in 289 msec 2024-11-15T21:55:48,248 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:55:48,252 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-15T21:55:48,252 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ae3311ea5ccb%2C32871%2C1731707679650.meta.1731707681806.meta, worker=ae3311ea5ccb,37455,1731707679725 in 302 msec 2024-11-15T21:55:48,254 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/WALs/ae3311ea5ccb,32871,1731707679650-splitting dir is empty, no logs to split. 2024-11-15T21:55:48,254 INFO [PEWorker-1 {}] master.SplitWALManager(105): ae3311ea5ccb,32871,1731707679650 WAL count=0, meta=true 2024-11-15T21:55:48,254 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if ae3311ea5ccb,32871,1731707679650 WAL splitting is done? wals=0, meta=true 2024-11-15T21:55:48,254 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-15T21:55:48,254 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN_META, hasLock=true; ServerCrashProcedure ae3311ea5ccb,32871,1731707679650, splitWal=true, meta=true 2024-11-15T21:55:48,254 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T21:55:48,254 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN_META, hasLock=true; ServerCrashProcedure ae3311ea5ccb,32871,1731707679650, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T21:55:48,265 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T21:55:48,265 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1] 2024-11-15T21:55:48,265 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to ae3311ea5ccb:32871 this server is in the failed servers list 2024-11-15T21:55:48,266 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=ae3311ea5ccb:32871 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:55:48,266 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:55:48,266 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 from cache 2024-11-15T21:55:48,575 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T21:55:48,575 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1] 2024-11-15T21:55:48,576 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to ae3311ea5ccb:32871 this server is in the failed servers list 2024-11-15T21:55:48,576 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=ae3311ea5ccb:32871 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:55:48,576 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:55:48,576 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 from cache 2024-11-15T21:55:49,085 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T21:55:49,085 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1] 2024-11-15T21:55:49,085 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to ae3311ea5ccb:32871 this server is in the failed servers list 2024-11-15T21:55:49,085 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=ae3311ea5ccb:32871 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:55:49,085 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:55:49,085 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 from cache 2024-11-15T21:55:50,094 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T21:55:50,094 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1] 2024-11-15T21:55:50,095 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server ae3311ea5ccb:32871 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T21:55:50,096 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,32871,1731707679650 for '' in 1588230740 of hbase:meta failed, tries = 6, maxAttempts = 46, timeout = 60000 ms, time elapsed = 2154 ms java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-15T21:55:50,096 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1, error=java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:55:50,096 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-15T21:55:50,096 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 from cache 2024-11-15T21:55:50,097 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address ae3311ea5ccb:32871 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:55:52,104 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T21:55:52,104 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1] 2024-11-15T21:55:52,105 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server ae3311ea5ccb:32871 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T21:55:52,106 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,32871,1731707679650 for '' in 1588230740 of hbase:meta failed, tries = 7, maxAttempts = 46, timeout = 60000 ms, time elapsed = 4163 ms java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-15T21:55:52,106 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1, error=java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:55:52,106 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-15T21:55:52,106 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 from cache 2024-11-15T21:55:52,106 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address ae3311ea5ccb:32871 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:55:56,124 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T21:55:56,124 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1] 2024-11-15T21:55:56,125 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server ae3311ea5ccb:32871 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T21:55:56,126 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,32871,1731707679650 for '' in 1588230740 of hbase:meta failed, tries = 8, maxAttempts = 46, timeout = 60000 ms, time elapsed = 8184 ms java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-15T21:55:56,126 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1, error=java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:55:56,126 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-15T21:55:56,126 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 from cache 2024-11-15T21:55:56,127 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address ae3311ea5ccb:32871 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:56:06,135 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T21:56:06,135 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1] 2024-11-15T21:56:06,136 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server ae3311ea5ccb:32871 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T21:56:06,137 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,32871,1731707679650 for '' in 1588230740 of hbase:meta failed, tries = 9, maxAttempts = 46, timeout = 60000 ms, time elapsed = 18195 ms java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-15T21:56:06,137 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1, error=java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:56:06,137 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-15T21:56:06,137 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 from cache 2024-11-15T21:56:06,138 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address ae3311ea5ccb:32871 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:56:07,961 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T21:56:16,145 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T21:56:16,145 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1] 2024-11-15T21:56:16,146 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server ae3311ea5ccb:32871 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T21:56:16,147 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,32871,1731707679650 for '' in 1588230740 of hbase:meta failed, tries = 10, maxAttempts = 46, timeout = 60000 ms, time elapsed = 28205 ms java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-15T21:56:16,147 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1, error=java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:56:16,147 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-15T21:56:16,147 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 from cache 2024-11-15T21:56:16,148 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address ae3311ea5ccb:32871 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:56:26,234 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T21:56:26,235 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1] 2024-11-15T21:56:26,235 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server ae3311ea5ccb:32871 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T21:56:26,236 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,32871,1731707679650 for '' in 1588230740 of hbase:meta failed, tries = 11, maxAttempts = 46, timeout = 60000 ms, time elapsed = 38294 ms java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-15T21:56:26,236 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1, error=java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:56:26,236 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-15T21:56:26,236 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 from cache 2024-11-15T21:56:26,237 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address ae3311ea5ccb:32871 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:56:36,315 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T21:56:36,315 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1] 2024-11-15T21:56:36,316 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server ae3311ea5ccb:32871 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T21:56:36,316 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,32871,1731707679650 for '' in 1588230740 of hbase:meta failed, tries = 12, maxAttempts = 46, timeout = 60000 ms, time elapsed = 48374 ms java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-15T21:56:36,317 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1, error=java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:56:36,317 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-15T21:56:36,317 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 from cache 2024-11-15T21:56:36,317 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address ae3311ea5ccb:32871 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:56:37,961 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T21:56:41,305 WARN [regionserver/ae3311ea5ccb:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 21, running: 1 2024-11-15T21:56:44,903 INFO [regionserver/ae3311ea5ccb:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactedHFilesCleaner was stopped 2024-11-15T21:56:47,945 WARN [Time-limited test {}] procedure.MasterProcedureTestingUtility$3(130): Failed to load meta org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=13, exceptions: 2024-11-15T21:55:47.944Z, java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:55:48.056Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=ae3311ea5ccb:32871 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:55:48.266Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=ae3311ea5ccb:32871 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:55:48.576Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=ae3311ea5ccb:32871 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:55:49.085Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=ae3311ea5ccb:32871 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:55:50.096Z, java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:55:52.106Z, java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:55:56.126Z, java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:56:06.137Z, java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:56:16.147Z, java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:56:26.236Z, java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:56:36.317Z, java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:493) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:461) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:422) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.fullScanRegions(MetaTableAccessor.java:108) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionStateStore.visitMeta(RegionStateStore.java:108) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.loadMeta(AssignmentManager.java:1907) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.joinCluster(AssignmentManager.java:1769) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:126) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:121) ~[test-classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:143) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:477) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:486) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPBase.testRecoveryAndDoubleExecution(TestSCPBase.java:114) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPWithMeta.testRecoveryAndDoubleExecutionOnRsWithMeta(TestSCPWithMeta.java:36) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:102) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-15T21:56:47,946 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 2 exec state=pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN_META, hasLock=true; ServerCrashProcedure ae3311ea5ccb,32871,1731707679650, splitWal=true, meta=true 2024-11-15T21:56:47,947 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T21:56:47,947 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T21:56:47,947 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T21:56:47,947 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T21:56:47,947 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T21:56:47,947 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T21:56:47,947 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T21:56:47,947 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T21:56:47,948 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T21:56:47,950 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T21:56:47,950 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T21:56:47,950 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T21:56:47,951 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T21:56:47,951 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-15T21:56:47,952 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, hasLock=false; ServerCrashProcedure ae3311ea5ccb,32871,1731707679650, splitWal=true, meta=true 2024-11-15T21:56:47,952 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ae3311ea5ccb%2C32871%2C1731707679650.meta.1731707681806.meta, worker=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:56:47,952 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ae3311ea5ccb%2C32871%2C1731707679650.meta.1731707681806.meta, worker=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:56:47,952 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, hasLock=false; ServerCrashProcedure ae3311ea5ccb,32871,1731707679650, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T21:56:47,952 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ae3311ea5ccb%2C32871%2C1731707679650.meta.1731707681806.meta, worker=ae3311ea5ccb,37455,1731707679725 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T21:56:47,952 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ae3311ea5ccb%2C32871%2C1731707679650.meta.1731707681806.meta, worker=ae3311ea5ccb,37455,1731707679725 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T21:56:47,952 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-15T21:56:47,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ae3311ea5ccb,32871,1731707679650, regionLocation=ae3311ea5ccb,32871,1731707679650, openSeqNum=2 2024-11-15T21:56:47,953 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ae3311ea5ccb,32871,1731707679650, table=hbase:meta, region=1588230740 2024-11-15T21:56:47,953 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ae3311ea5ccb,32871,1731707679650, state=OPEN 2024-11-15T21:56:47,955 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37455-0x100f86521fe0003, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T21:56:47,955 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40941-0x100f86521fe0001, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T21:56:47,955 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36531-0x100f86521fe0000, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T21:56:47,955 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T21:56:47,955 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T21:56:47,955 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T21:56:47,957 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731707837957 2024-11-15T21:56:47,957 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-15T21:56:47,958 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T21:56:47,958 DEBUG [Time-limited test {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1] 2024-11-15T21:56:47,959 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server ae3311ea5ccb:32871 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T21:56:47,959 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1, error=java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:56:47,960 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-15T21:56:47,960 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 from cache 2024-11-15T21:56:47,960 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address ae3311ea5ccb:32871 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:56:47,962 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/WALs/ae3311ea5ccb,32871,1731707679650-splitting dir is empty, no logs to split. 2024-11-15T21:56:47,962 INFO [PEWorker-1 {}] master.SplitWALManager(105): ae3311ea5ccb,32871,1731707679650 WAL count=0, meta=true 2024-11-15T21:56:47,962 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if ae3311ea5ccb,32871,1731707679650 WAL splitting is done? wals=0, meta=true 2024-11-15T21:56:47,962 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-15T21:56:47,964 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-15T21:56:47,965 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-15T21:56:47,965 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_GET_REGIONS, hasLock=true; ServerCrashProcedure ae3311ea5ccb,32871,1731707679650, splitWal=true, meta=true 2024-11-15T21:56:47,965 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T21:56:47,965 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_GET_REGIONS, hasLock=true; ServerCrashProcedure ae3311ea5ccb,32871,1731707679650, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T21:56:48,064 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T21:56:48,064 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1] 2024-11-15T21:56:48,064 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to ae3311ea5ccb:32871 this server is in the failed servers list 2024-11-15T21:56:48,065 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=ae3311ea5ccb:32871 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:56:48,065 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:56:48,065 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 from cache 2024-11-15T21:56:48,274 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T21:56:48,275 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1] 2024-11-15T21:56:48,275 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to ae3311ea5ccb:32871 this server is in the failed servers list 2024-11-15T21:56:48,275 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=ae3311ea5ccb:32871 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:56:48,275 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:56:48,275 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 from cache 2024-11-15T21:56:48,585 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T21:56:48,585 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1] 2024-11-15T21:56:48,585 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to ae3311ea5ccb:32871 this server is in the failed servers list 2024-11-15T21:56:48,586 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=ae3311ea5ccb:32871 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:56:48,586 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:56:48,586 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 from cache 2024-11-15T21:56:49,095 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T21:56:49,095 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1] 2024-11-15T21:56:49,095 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to ae3311ea5ccb:32871 this server is in the failed servers list 2024-11-15T21:56:49,096 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=ae3311ea5ccb:32871 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:56:49,096 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:56:49,096 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 from cache 2024-11-15T21:56:50,114 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T21:56:50,114 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1] 2024-11-15T21:56:50,115 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server ae3311ea5ccb:32871 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T21:56:50,116 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,32871,1731707679650 for '' in 1588230740 of hbase:meta failed, tries = 6, maxAttempts = 46, timeout = 60000 ms, time elapsed = 2158 ms java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-15T21:56:50,116 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1, error=java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:56:50,116 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-15T21:56:50,116 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 from cache 2024-11-15T21:56:50,117 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address ae3311ea5ccb:32871 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:56:52,135 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T21:56:52,135 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1] 2024-11-15T21:56:52,136 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server ae3311ea5ccb:32871 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T21:56:52,136 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,32871,1731707679650 for '' in 1588230740 of hbase:meta failed, tries = 7, maxAttempts = 46, timeout = 60000 ms, time elapsed = 4178 ms java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-15T21:56:52,137 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1, error=java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:56:52,137 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-15T21:56:52,137 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 from cache 2024-11-15T21:56:52,137 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address ae3311ea5ccb:32871 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:56:52,803 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=ae3311ea5ccb:36531 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] ... 10 more 2024-11-15T21:56:56,174 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T21:56:56,174 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1] 2024-11-15T21:56:56,175 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server ae3311ea5ccb:32871 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T21:56:56,176 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,32871,1731707679650 for '' in 1588230740 of hbase:meta failed, tries = 8, maxAttempts = 46, timeout = 60000 ms, time elapsed = 8218 ms java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-15T21:56:56,176 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1, error=java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:56:56,176 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-15T21:56:56,177 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 from cache 2024-11-15T21:56:56,177 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address ae3311ea5ccb:32871 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:57:06,215 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T21:57:06,215 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1] 2024-11-15T21:57:06,216 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server ae3311ea5ccb:32871 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T21:57:06,217 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,32871,1731707679650 for '' in 1588230740 of hbase:meta failed, tries = 9, maxAttempts = 46, timeout = 60000 ms, time elapsed = 18258 ms java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-15T21:57:06,217 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1, error=java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:57:06,217 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-15T21:57:06,217 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 from cache 2024-11-15T21:57:06,217 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address ae3311ea5ccb:32871 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:57:07,961 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T21:57:16,264 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T21:57:16,265 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1] 2024-11-15T21:57:16,266 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server ae3311ea5ccb:32871 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T21:57:16,266 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,32871,1731707679650 for '' in 1588230740 of hbase:meta failed, tries = 10, maxAttempts = 46, timeout = 60000 ms, time elapsed = 28308 ms java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-15T21:57:16,267 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1, error=java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:57:16,267 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-15T21:57:16,267 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 from cache 2024-11-15T21:57:16,267 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address ae3311ea5ccb:32871 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:57:26,284 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T21:57:26,285 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1] 2024-11-15T21:57:26,286 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server ae3311ea5ccb:32871 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T21:57:26,287 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,32871,1731707679650 for '' in 1588230740 of hbase:meta failed, tries = 11, maxAttempts = 46, timeout = 60000 ms, time elapsed = 38328 ms java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-15T21:57:26,287 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1, error=java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:57:26,287 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-15T21:57:26,287 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 from cache 2024-11-15T21:57:26,288 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address ae3311ea5ccb:32871 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:57:35,280 WARN [RS:0;ae3311ea5ccb:40941 {}] monitoring.TaskMonitor(166): Status Processing ServerCrashProcedure of ae3311ea5ccb,32871,1731707679650: status=Processing ServerCrashProcedure of ae3311ea5ccb,32871,1731707679650 current State SERVER_CRASH_SPLIT_META_LOGS, state=RUNNING, startTime=1731707687904, completionTime=-1 appears to have been leaked 2024-11-15T21:57:35,280 WARN [RS:0;ae3311ea5ccb:40941 {}] monitoring.TaskMonitor(166): Status Processing ServerCrashProcedure of ae3311ea5ccb,32871,1731707679650: status=Processing ServerCrashProcedure of ae3311ea5ccb,32871,1731707679650 current State SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, state=RUNNING, startTime=1731707747932, completionTime=-1 appears to have been leaked 2024-11-15T21:57:36,305 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T21:57:36,305 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1] 2024-11-15T21:57:36,306 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server ae3311ea5ccb:32871 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T21:57:36,307 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,32871,1731707679650 for '' in 1588230740 of hbase:meta failed, tries = 12, maxAttempts = 46, timeout = 60000 ms, time elapsed = 48349 ms java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-15T21:57:36,308 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1, error=java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:57:36,308 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-15T21:57:36,308 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 from cache 2024-11-15T21:57:36,308 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address ae3311ea5ccb:32871 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:57:37,962 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T21:57:47,966 WARN [Time-limited test {}] procedure.MasterProcedureTestingUtility$3(130): Failed to load meta org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=13, exceptions: 2024-11-15T21:56:47.960Z, java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:56:48.065Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=ae3311ea5ccb:32871 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:56:48.275Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=ae3311ea5ccb:32871 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:56:48.586Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=ae3311ea5ccb:32871 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:56:49.096Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=ae3311ea5ccb:32871 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:56:50.116Z, java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:56:52.137Z, java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:56:56.177Z, java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:57:06.217Z, java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:57:16.267Z, java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:57:26.287Z, java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:57:36.308Z, java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:493) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:461) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:422) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.fullScanRegions(MetaTableAccessor.java:108) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionStateStore.visitMeta(RegionStateStore.java:108) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.loadMeta(AssignmentManager.java:1907) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.joinCluster(AssignmentManager.java:1769) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:126) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:121) ~[test-classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:143) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:477) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:486) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPBase.testRecoveryAndDoubleExecution(TestSCPBase.java:114) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPWithMeta.testRecoveryAndDoubleExecutionOnRsWithMeta(TestSCPWithMeta.java:36) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:102) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-15T21:57:47,967 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 3 exec state=pid=57, state=WAITING:SERVER_CRASH_GET_REGIONS, hasLock=true; ServerCrashProcedure ae3311ea5ccb,32871,1731707679650, splitWal=true, meta=true 2024-11-15T21:57:47,967 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T21:57:47,967 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T21:57:47,967 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T21:57:47,967 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T21:57:47,968 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T21:57:47,968 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T21:57:47,968 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T21:57:47,968 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T21:57:47,969 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T21:57:47,971 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T21:57:47,971 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T21:57:47,971 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T21:57:47,971 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T21:57:47,972 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-15T21:57:47,972 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN_META, hasLock=false; ServerCrashProcedure ae3311ea5ccb,32871,1731707679650, splitWal=true, meta=true 2024-11-15T21:57:47,972 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ae3311ea5ccb%2C32871%2C1731707679650.meta.1731707681806.meta, worker=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:57:47,973 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ae3311ea5ccb%2C32871%2C1731707679650.meta.1731707681806.meta, worker=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:57:47,973 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN_META, hasLock=false; ServerCrashProcedure ae3311ea5ccb,32871,1731707679650, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T21:57:47,973 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ae3311ea5ccb%2C32871%2C1731707679650.meta.1731707681806.meta, worker=ae3311ea5ccb,37455,1731707679725 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T21:57:47,973 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ae3311ea5ccb%2C32871%2C1731707679650.meta.1731707681806.meta, worker=ae3311ea5ccb,37455,1731707679725 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T21:57:47,973 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-15T21:57:47,974 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ae3311ea5ccb,32871,1731707679650, regionLocation=ae3311ea5ccb,32871,1731707679650, openSeqNum=2 2024-11-15T21:57:47,974 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ae3311ea5ccb,32871,1731707679650, table=hbase:meta, region=1588230740 2024-11-15T21:57:47,974 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ae3311ea5ccb,32871,1731707679650, state=OPEN 2024-11-15T21:57:47,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40941-0x100f86521fe0001, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T21:57:47,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37455-0x100f86521fe0003, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T21:57:47,977 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T21:57:47,977 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T21:57:47,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36531-0x100f86521fe0000, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T21:57:47,978 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T21:57:47,985 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731707897985 2024-11-15T21:57:47,985 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-15T21:57:47,985 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-15T21:57:47,985 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-15T21:57:47,985 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T21:57:47,986 DEBUG [Time-limited test {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1] 2024-11-15T21:57:47,987 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server ae3311ea5ccb:32871 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T21:57:47,987 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1, error=java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:57:47,987 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-15T21:57:47,987 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 from cache 2024-11-15T21:57:47,988 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address ae3311ea5ccb:32871 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:57:47,988 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T21:57:47,989 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-15T21:57:48,094 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T21:57:48,095 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1] 2024-11-15T21:57:48,095 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to ae3311ea5ccb:32871 this server is in the failed servers list 2024-11-15T21:57:48,095 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=ae3311ea5ccb:32871 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:57:48,096 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:57:48,096 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 from cache 2024-11-15T21:57:48,140 DEBUG [ae3311ea5ccb:36531 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-15T21:57:48,141 DEBUG [ae3311ea5ccb:36531 {}] balancer.BalancerClusterState(204): Hosts are {ae3311ea5ccb=0} racks are {/default-rack=0} 2024-11-15T21:57:48,141 DEBUG [ae3311ea5ccb:36531 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-15T21:57:48,141 DEBUG [ae3311ea5ccb:36531 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-15T21:57:48,141 DEBUG [ae3311ea5ccb:36531 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-15T21:57:48,141 DEBUG [ae3311ea5ccb:36531 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-15T21:57:48,141 INFO [ae3311ea5ccb:36531 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-15T21:57:48,141 INFO [ae3311ea5ccb:36531 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-15T21:57:48,141 DEBUG [ae3311ea5ccb:36531 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-15T21:57:48,142 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ae3311ea5ccb,40941,1731707679516 2024-11-15T21:57:48,143 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ae3311ea5ccb,40941,1731707679516, state=OPENING 2024-11-15T21:57:48,145 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37455-0x100f86521fe0003, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T21:57:48,145 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40941-0x100f86521fe0001, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T21:57:48,145 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36531-0x100f86521fe0000, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T21:57:48,145 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T21:57:48,145 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T21:57:48,145 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T21:57:48,145 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T21:57:48,146 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=ae3311ea5ccb,40941,1731707679516}] 2024-11-15T21:57:48,301 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37129, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-15T21:57:48,305 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T21:57:48,306 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,40941,1731707679516, seqNum=-1] 2024-11-15T21:57:48,306 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T21:57:48,309 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55993, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T21:57:48,313 INFO [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-15T21:57:48,313 INFO [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T21:57:48,314 INFO [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=61}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-15T21:57:48,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40941 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Scan size: 121 connection: 172.17.0.3:55993 deadline: 1731707927989, exception=org.apache.hadoop.hbase.NotServingRegionException: hbase:meta,,1 is not online on ae3311ea5ccb,40941,1731707679516 2024-11-15T21:57:48,316 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,40941,1731707679516, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,40941,1731707679516, seqNum=-1, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: hbase:meta,,1 is not online on ae3311ea5ccb,40941,1731707679516 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T21:57:48,316 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,40941,1731707679516, seqNum=-1 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: hbase:meta,,1 is not online on ae3311ea5ccb,40941,1731707679516 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T21:57:48,316 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,40941,1731707679516, seqNum=-1 from cache 2024-11-15T21:57:48,327 INFO [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ae3311ea5ccb%2C40941%2C1731707679516.meta, suffix=.meta, logDir=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/WALs/ae3311ea5ccb,40941,1731707679516, archiveDir=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/oldWALs, maxLogs=32 2024-11-15T21:57:48,344 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=61}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/WALs/ae3311ea5ccb,40941,1731707679516/ae3311ea5ccb%2C40941%2C1731707679516.meta.1731707868328.meta, exclude list is [], retry=0 2024-11-15T21:57:48,354 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40887,DS-cfe332f0-c324-40d1-87f6-7313ac5c3921,DISK] 2024-11-15T21:57:48,355 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36461,DS-34745d98-d41f-404b-baa7-624040cb25c7,DISK] 2024-11-15T21:57:48,357 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41653,DS-f55f5b18-f6a1-4c42-97ca-ec4b858b4b31,DISK] 2024-11-15T21:57:48,366 INFO [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/WALs/ae3311ea5ccb,40941,1731707679516/ae3311ea5ccb%2C40941%2C1731707679516.meta.1731707868328.meta 2024-11-15T21:57:48,366 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44379:44379),(127.0.0.1/127.0.0.1:46569:46569),(127.0.0.1/127.0.0.1:41837:41837)] 2024-11-15T21:57:48,367 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-15T21:57:48,368 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=61}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-15T21:57:48,368 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-15T21:57:48,368 INFO [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-15T21:57:48,368 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-15T21:57:48,368 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:57:48,369 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-15T21:57:48,369 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-15T21:57:48,371 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-15T21:57:48,372 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-15T21:57:48,372 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:57:48,372 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T21:57:48,373 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-15T21:57:48,374 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-15T21:57:48,374 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:57:48,375 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T21:57:48,375 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-15T21:57:48,376 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-15T21:57:48,376 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:57:48,377 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T21:57:48,377 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-15T21:57:48,378 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-15T21:57:48,378 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:57:48,379 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T21:57:48,379 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-15T21:57:48,380 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/hbase/meta/1588230740 2024-11-15T21:57:48,383 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/hbase/meta/1588230740 2024-11-15T21:57:48,385 INFO [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-15T21:57:48,407 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5793): Applied 264, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=60, path=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-15T21:57:48,412 INFO [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=49.58 KB heapSize=69.98 KB 2024-11-15T21:57:48,502 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/hbase/meta/1588230740/.tmp/info/687b3c5a2a2c4758b7d48605b305c703 is 268, key is testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731707682333.dd723478fcfc0e25b606965c9135244b./info:regioninfo/1731707684278/Put/seqid=0 2024-11-15T21:57:48,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741866_1042 (size=58053) 2024-11-15T21:57:48,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741866_1042 (size=58053) 2024-11-15T21:57:48,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741866_1042 (size=58053) 2024-11-15T21:57:48,550 INFO [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.31 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/hbase/meta/1588230740/.tmp/info/687b3c5a2a2c4758b7d48605b305c703 2024-11-15T21:57:48,624 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T21:57:48,625 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,40941,1731707679516, seqNum=-1] 2024-11-15T21:57:48,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40941 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Scan size: 121 connection: 172.17.0.3:55993 deadline: 1731707927986, exception=org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on ae3311ea5ccb,40941,1731707679516 2024-11-15T21:57:48,628 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,40941,1731707679516, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,40941,1731707679516, seqNum=-1, error=org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on ae3311ea5ccb,40941,1731707679516 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T21:57:48,628 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,40941,1731707679516, seqNum=-1 is org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on ae3311ea5ccb,40941,1731707679516 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T21:57:48,628 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,40941,1731707679516, seqNum=-1 from cache 2024-11-15T21:57:48,643 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/hbase/meta/1588230740/.tmp/ns/cf9864883c49425ca981ab57b59c2aba is 43, key is default/ns:d/1731707681999/Put/seqid=0 2024-11-15T21:57:48,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741867_1043 (size=5153) 2024-11-15T21:57:48,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741867_1043 (size=5153) 2024-11-15T21:57:48,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741867_1043 (size=5153) 2024-11-15T21:57:48,660 INFO [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/hbase/meta/1588230740/.tmp/ns/cf9864883c49425ca981ab57b59c2aba 2024-11-15T21:57:48,697 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/hbase/meta/1588230740/.tmp/table/cc976aa47363475d8c8c9d515f1f21ef is 105, key is testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/table:state/1731707684290/Put/seqid=0 2024-11-15T21:57:48,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741868_1044 (size=5620) 2024-11-15T21:57:48,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741868_1044 (size=5620) 2024-11-15T21:57:48,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741868_1044 (size=5620) 2024-11-15T21:57:48,726 INFO [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=202 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/hbase/meta/1588230740/.tmp/table/cc976aa47363475d8c8c9d515f1f21ef 2024-11-15T21:57:48,742 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/hbase/meta/1588230740/.tmp/info/687b3c5a2a2c4758b7d48605b305c703 as hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/hbase/meta/1588230740/info/687b3c5a2a2c4758b7d48605b305c703 2024-11-15T21:57:48,762 INFO [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/hbase/meta/1588230740/info/687b3c5a2a2c4758b7d48605b305c703, entries=260, sequenceid=60, filesize=56.7 K 2024-11-15T21:57:48,766 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/hbase/meta/1588230740/.tmp/ns/cf9864883c49425ca981ab57b59c2aba as hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/hbase/meta/1588230740/ns/cf9864883c49425ca981ab57b59c2aba 2024-11-15T21:57:48,791 INFO [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/hbase/meta/1588230740/ns/cf9864883c49425ca981ab57b59c2aba, entries=2, sequenceid=60, filesize=5.0 K 2024-11-15T21:57:48,794 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/hbase/meta/1588230740/.tmp/table/cc976aa47363475d8c8c9d515f1f21ef as hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/hbase/meta/1588230740/table/cc976aa47363475d8c8c9d515f1f21ef 2024-11-15T21:57:48,815 INFO [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/hbase/meta/1588230740/table/cc976aa47363475d8c8c9d515f1f21ef, entries=2, sequenceid=60, filesize=5.5 K 2024-11-15T21:57:48,820 INFO [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~49.58 KB/50772, heapSize ~69.69 KB/71360, currentSize=0 B/0 for 1588230740 in 407ms, sequenceid=60, compaction requested=false; wal=null 2024-11-15T21:57:48,822 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-15T21:57:48,824 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-15T21:57:48,824 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-15T21:57:48,825 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-15T21:57:48,828 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-15T21:57:48,831 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/hbase/meta/1588230740/recovered.edits/60.seqid, newMaxSeqId=60, maxSeqId=1 2024-11-15T21:57:48,832 INFO [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=61; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67996154, jitterRate=0.013221651315689087}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-15T21:57:48,832 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-15T21:57:48,833 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731707868369Writing region info on filesystem at 1731707868369Initializing all the Stores at 1731707868370 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731707868370Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731707868370Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707868370Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731707868370Obtaining lock to block concurrent updates at 1731707868412 (+42 ms)Preparing flush snapshotting stores in 1588230740 at 1731707868412Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=50772, getHeapSize=71600, getOffHeapSize=0, getCellsCount=264 at 1731707868416 (+4 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731707868416Flushing 1588230740/info: creating writer at 1731707868420 (+4 ms)Flushing 1588230740/info: appending metadata at 1731707868493 (+73 ms)Flushing 1588230740/info: closing flushed file at 1731707868497 (+4 ms)Flushing 1588230740/ns: creating writer at 1731707868618 (+121 ms)Flushing 1588230740/ns: appending metadata at 1731707868641 (+23 ms)Flushing 1588230740/ns: closing flushed file at 1731707868641Flushing 1588230740/table: creating writer at 1731707868671 (+30 ms)Flushing 1588230740/table: appending metadata at 1731707868696 (+25 ms)Flushing 1588230740/table: closing flushed file at 1731707868696Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a086a19: reopening flushed file at 1731707868739 (+43 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e7285c4: reopening flushed file at 1731707868764 (+25 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19f47a5d: reopening flushed file at 1731707868791 (+27 ms)Finished flush of dataSize ~49.58 KB/50772, heapSize ~69.69 KB/71360, currentSize=0 B/0 for 1588230740 in 407ms, sequenceid=60, compaction requested=false; wal=null at 1731707868820 (+29 ms)Cleaning up temporary data from old regions at 1731707868824 (+4 ms)Running coprocessor post-open hooks at 1731707868832 (+8 ms)Region opened successfully at 1731707868832 2024-11-15T21:57:48,837 INFO [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=61, masterSystemTime=1731707868298 2024-11-15T21:57:48,842 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-15T21:57:48,842 INFO [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-15T21:57:48,843 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=61, regionLocation=ae3311ea5ccb,40941,1731707679516 2024-11-15T21:57:48,846 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ae3311ea5ccb,40941,1731707679516, state=OPEN 2024-11-15T21:57:48,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40941-0x100f86521fe0001, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T21:57:48,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37455-0x100f86521fe0003, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T21:57:48,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36531-0x100f86521fe0000, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T21:57:48,848 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T21:57:48,848 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T21:57:48,848 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T21:57:48,849 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=61, ppid=60, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=ae3311ea5ccb,40941,1731707679516 2024-11-15T21:57:48,853 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-15T21:57:48,853 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ae3311ea5ccb,40941,1731707679516 in 704 msec 2024-11-15T21:57:48,856 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=57 2024-11-15T21:57:48,856 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 868 msec 2024-11-15T21:57:49,134 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T21:57:49,135 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,40941,1731707679516, seqNum=-1] 2024-11-15T21:57:49,172 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=229a036fd38ae13e7359b15bc2f2e587, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=2 2024-11-15T21:57:49,173 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dd723478fcfc0e25b606965c9135244b, regionState=OPEN, lastHost=ae3311ea5ccb,32871,1731707679650, regionLocation=ae3311ea5ccb,32871,1731707679650, openSeqNum=2 2024-11-15T21:57:49,173 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50df10b3ae47c5dc7c4521c4c58c192f, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=2 2024-11-15T21:57:49,173 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4256fff2fa33299a6264dd460a4003fb, regionState=OPEN, lastHost=ae3311ea5ccb,32871,1731707679650, regionLocation=ae3311ea5ccb,32871,1731707679650, openSeqNum=2 2024-11-15T21:57:49,173 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2c83200b2c422973a68afce1173aae30, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=2 2024-11-15T21:57:49,173 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f13d1331edcf64c426c3bf9433330e4, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=2 2024-11-15T21:57:49,173 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9ef867a298d175eca9d85304f2faa69, regionState=OPEN, lastHost=ae3311ea5ccb,32871,1731707679650, regionLocation=ae3311ea5ccb,32871,1731707679650, openSeqNum=2 2024-11-15T21:57:49,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0890aeec1f0ca0649e6483a2d56f36b4, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=2 2024-11-15T21:57:49,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d28ee8a2ac06d2199c72ac7f884ba470, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=2 2024-11-15T21:57:49,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aec2e46cb64898ba2a66825d3732f6dc, regionState=OPEN, lastHost=ae3311ea5ccb,32871,1731707679650, regionLocation=ae3311ea5ccb,32871,1731707679650, openSeqNum=2 2024-11-15T21:57:49,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cbb6b5b8341f7ff2ada4ac1931a1a588, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=2 2024-11-15T21:57:49,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a3f7be24d811f4d6177c28e08f25af2d, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=2 2024-11-15T21:57:49,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e3b749fcbfdadfe59f1d1cf94f5406a, regionState=OPEN, lastHost=ae3311ea5ccb,32871,1731707679650, regionLocation=ae3311ea5ccb,32871,1731707679650, openSeqNum=2 2024-11-15T21:57:49,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51af260ba7a9b40a7f788a5c93a416b6, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=2 2024-11-15T21:57:49,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ab2d104e772f41ee9c56851adad43f0, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=2 2024-11-15T21:57:49,175 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3cf264afea186a5cc96edd3e68a10d45, regionState=OPEN, lastHost=ae3311ea5ccb,32871,1731707679650, regionLocation=ae3311ea5ccb,32871,1731707679650, openSeqNum=2 2024-11-15T21:57:49,175 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d100add17d87270d000a3c39b49f8227, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=2 2024-11-15T21:57:49,175 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d9989b0341ef96818e811f874dbcf91c, regionState=OPEN, lastHost=ae3311ea5ccb,32871,1731707679650, regionLocation=ae3311ea5ccb,32871,1731707679650, openSeqNum=2 2024-11-15T21:57:49,175 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=386f4af1e738f142f3fbde37773d4d2c, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=2 2024-11-15T21:57:49,175 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d6063c53955b74f03cc36bd578a6bd41, regionState=OPEN, lastHost=ae3311ea5ccb,32871,1731707679650, regionLocation=ae3311ea5ccb,32871,1731707679650, openSeqNum=2 2024-11-15T21:57:49,175 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fac6af87aa0fc9826df0515f766c031e, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=2 2024-11-15T21:57:49,175 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ce283f02f0974e39c98759bb2bc9883, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=2 2024-11-15T21:57:49,176 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6694325f9c862a5a1ee02eabf7861cc3, regionState=OPEN, lastHost=ae3311ea5ccb,32871,1731707679650, regionLocation=ae3311ea5ccb,32871,1731707679650, openSeqNum=2 2024-11-15T21:57:49,176 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46b5f83abe9585829b26884a1b26b43f, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=2 2024-11-15T21:57:49,176 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1031049c0b4149d1189db33e38ab56cd, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=2 2024-11-15T21:57:49,176 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e76ae188febdca91d319999f619d9e11, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=2 2024-11-15T21:57:49,176 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-15T21:57:49,176 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731707929176 2024-11-15T21:57:49,176 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731707989176 2024-11-15T21:57:49,176 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 1.1910 sec 2024-11-15T21:57:49,177 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): ae3311ea5ccb,32871,1731707679650 had 10 regions 2024-11-15T21:57:49,177 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-15T21:57:49,177 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure ae3311ea5ccb,32871,1731707679650, splitWal=true, meta=true 2024-11-15T21:57:49,177 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T21:57:49,177 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure ae3311ea5ccb,32871,1731707679650, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T21:57:49,427 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 4 exec state=pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure ae3311ea5ccb,32871,1731707679650, splitWal=true, meta=true 2024-11-15T21:57:49,427 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T21:57:49,427 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T21:57:49,427 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T21:57:49,427 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T21:57:49,427 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T21:57:49,428 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T21:57:49,428 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T21:57:49,428 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T21:57:49,429 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-15T21:57:49,431 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T21:57:49,431 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T21:57:49,431 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T21:57:49,431 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T21:57:49,432 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T21:57:49,432 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T21:57:49,432 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-15T21:57:49,437 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_GET_REGIONS, hasLock=false; ServerCrashProcedure ae3311ea5ccb,32871,1731707679650, splitWal=true, meta=true 2024-11-15T21:57:49,437 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ae3311ea5ccb%2C32871%2C1731707679650.meta.1731707681806.meta, worker=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:57:49,437 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ae3311ea5ccb%2C32871%2C1731707679650.meta.1731707681806.meta, worker=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:57:49,437 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T21:57:49,437 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ae3311ea5ccb,40941,1731707679516 2024-11-15T21:57:49,437 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_GET_REGIONS, hasLock=false; ServerCrashProcedure ae3311ea5ccb,32871,1731707679650, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T21:57:49,437 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ae3311ea5ccb%2C32871%2C1731707679650.meta.1731707681806.meta, worker=ae3311ea5ccb,37455,1731707679725 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T21:57:49,438 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ae3311ea5ccb%2C32871%2C1731707679650.meta.1731707681806.meta, worker=ae3311ea5ccb,37455,1731707679725 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T21:57:49,438 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T21:57:49,438 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ae3311ea5ccb,40941,1731707679516 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T21:57:49,438 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-11-15T21:57:49,453 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=61 2024-11-15T21:57:49,453 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ae3311ea5ccb,40941,1731707679516, table=hbase:meta, region=1588230740 2024-11-15T21:57:49,454 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ae3311ea5ccb,40941,1731707679516, state=OPEN 2024-11-15T21:57:49,461 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40941-0x100f86521fe0001, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T21:57:49,461 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T21:57:49,465 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36531-0x100f86521fe0000, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T21:57:49,465 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37455-0x100f86521fe0003, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T21:57:49,465 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T21:57:49,465 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T21:57:49,471 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731707899471 2024-11-15T21:57:49,472 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-15T21:57:49,514 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=229a036fd38ae13e7359b15bc2f2e587, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=2 2024-11-15T21:57:49,514 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dd723478fcfc0e25b606965c9135244b, regionState=OPEN, lastHost=ae3311ea5ccb,32871,1731707679650, regionLocation=ae3311ea5ccb,32871,1731707679650, openSeqNum=2 2024-11-15T21:57:49,514 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50df10b3ae47c5dc7c4521c4c58c192f, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=2 2024-11-15T21:57:49,515 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4256fff2fa33299a6264dd460a4003fb, regionState=OPEN, lastHost=ae3311ea5ccb,32871,1731707679650, regionLocation=ae3311ea5ccb,32871,1731707679650, openSeqNum=2 2024-11-15T21:57:49,515 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2c83200b2c422973a68afce1173aae30, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=2 2024-11-15T21:57:49,515 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f13d1331edcf64c426c3bf9433330e4, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=2 2024-11-15T21:57:49,515 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9ef867a298d175eca9d85304f2faa69, regionState=OPEN, lastHost=ae3311ea5ccb,32871,1731707679650, regionLocation=ae3311ea5ccb,32871,1731707679650, openSeqNum=2 2024-11-15T21:57:49,516 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0890aeec1f0ca0649e6483a2d56f36b4, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=2 2024-11-15T21:57:49,516 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d28ee8a2ac06d2199c72ac7f884ba470, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=2 2024-11-15T21:57:49,516 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aec2e46cb64898ba2a66825d3732f6dc, regionState=OPEN, lastHost=ae3311ea5ccb,32871,1731707679650, regionLocation=ae3311ea5ccb,32871,1731707679650, openSeqNum=2 2024-11-15T21:57:49,516 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cbb6b5b8341f7ff2ada4ac1931a1a588, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=2 2024-11-15T21:57:49,516 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a3f7be24d811f4d6177c28e08f25af2d, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=2 2024-11-15T21:57:49,517 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e3b749fcbfdadfe59f1d1cf94f5406a, regionState=OPEN, lastHost=ae3311ea5ccb,32871,1731707679650, regionLocation=ae3311ea5ccb,32871,1731707679650, openSeqNum=2 2024-11-15T21:57:49,517 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51af260ba7a9b40a7f788a5c93a416b6, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=2 2024-11-15T21:57:49,517 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ab2d104e772f41ee9c56851adad43f0, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=2 2024-11-15T21:57:49,517 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3cf264afea186a5cc96edd3e68a10d45, regionState=OPEN, lastHost=ae3311ea5ccb,32871,1731707679650, regionLocation=ae3311ea5ccb,32871,1731707679650, openSeqNum=2 2024-11-15T21:57:49,517 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d100add17d87270d000a3c39b49f8227, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=2 2024-11-15T21:57:49,518 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d9989b0341ef96818e811f874dbcf91c, regionState=OPEN, lastHost=ae3311ea5ccb,32871,1731707679650, regionLocation=ae3311ea5ccb,32871,1731707679650, openSeqNum=2 2024-11-15T21:57:49,518 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=386f4af1e738f142f3fbde37773d4d2c, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=2 2024-11-15T21:57:49,518 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d6063c53955b74f03cc36bd578a6bd41, regionState=OPEN, lastHost=ae3311ea5ccb,32871,1731707679650, regionLocation=ae3311ea5ccb,32871,1731707679650, openSeqNum=2 2024-11-15T21:57:49,519 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fac6af87aa0fc9826df0515f766c031e, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=2 2024-11-15T21:57:49,519 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ce283f02f0974e39c98759bb2bc9883, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=2 2024-11-15T21:57:49,519 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6694325f9c862a5a1ee02eabf7861cc3, regionState=OPEN, lastHost=ae3311ea5ccb,32871,1731707679650, regionLocation=ae3311ea5ccb,32871,1731707679650, openSeqNum=2 2024-11-15T21:57:49,519 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46b5f83abe9585829b26884a1b26b43f, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=2 2024-11-15T21:57:49,519 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1031049c0b4149d1189db33e38ab56cd, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=2 2024-11-15T21:57:49,519 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e76ae188febdca91d319999f619d9e11, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=2 2024-11-15T21:57:49,519 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-15T21:57:49,520 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731707929520 2024-11-15T21:57:49,520 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731707989520 2024-11-15T21:57:49,520 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 48 msec 2024-11-15T21:57:49,522 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): ae3311ea5ccb,32871,1731707679650 had 9 regions 2024-11-15T21:57:49,522 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-15T21:57:49,535 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure ae3311ea5ccb,32871,1731707679650, splitWal=true, meta=true, isMeta: false 2024-11-15T21:57:49,556 INFO [PEWorker-1 {}] master.SplitWALManager(105): ae3311ea5ccb,32871,1731707679650 WAL count=1, meta=false 2024-11-15T21:57:49,556 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure ae3311ea5ccb%2C32871%2C1731707679650.1731707681383}] 2024-11-15T21:57:49,556 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-15T21:57:49,556 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_WALS_DIR, hasLock=true; ServerCrashProcedure ae3311ea5ccb,32871,1731707679650, splitWal=true, meta=true 2024-11-15T21:57:49,556 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T21:57:49,557 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_WALS_DIR, hasLock=true; ServerCrashProcedure ae3311ea5ccb,32871,1731707679650, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T21:57:49,771 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 5 exec state=pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_WALS_DIR, hasLock=true; ServerCrashProcedure ae3311ea5ccb,32871,1731707679650, splitWal=true, meta=true 2024-11-15T21:57:49,771 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T21:57:49,771 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T21:57:49,771 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T21:57:49,772 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T21:57:49,772 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T21:57:49,772 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T21:57:49,772 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T21:57:49,772 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T21:57:49,774 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-15T21:57:49,775 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T21:57:49,775 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T21:57:49,776 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T21:57:49,776 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T21:57:49,776 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T21:57:49,776 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T21:57:49,776 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-15T21:57:49,777 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=false; ServerCrashProcedure ae3311ea5ccb,32871,1731707679650, splitWal=true, meta=true 2024-11-15T21:57:49,777 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ae3311ea5ccb%2C32871%2C1731707679650.meta.1731707681806.meta, worker=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:57:49,777 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ae3311ea5ccb%2C32871%2C1731707679650.meta.1731707681806.meta, worker=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:57:49,777 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T21:57:49,778 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ae3311ea5ccb,40941,1731707679516 2024-11-15T21:57:49,778 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=false; ServerCrashProcedure ae3311ea5ccb,32871,1731707679650, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T21:57:49,778 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ae3311ea5ccb%2C32871%2C1731707679650.meta.1731707681806.meta, worker=ae3311ea5ccb,37455,1731707679725 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T21:57:49,778 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ae3311ea5ccb%2C32871%2C1731707679650.meta.1731707681806.meta, worker=ae3311ea5ccb,37455,1731707679725 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T21:57:49,778 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T21:57:49,778 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ae3311ea5ccb,40941,1731707679516 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T21:57:49,778 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-15T21:57:49,779 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=61 2024-11-15T21:57:49,779 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ae3311ea5ccb,40941,1731707679516, table=hbase:meta, region=1588230740 2024-11-15T21:57:49,779 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ae3311ea5ccb,40941,1731707679516, state=OPEN 2024-11-15T21:57:49,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37455-0x100f86521fe0003, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T21:57:49,784 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T21:57:49,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36531-0x100f86521fe0000, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T21:57:49,785 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T21:57:49,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40941-0x100f86521fe0001, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T21:57:49,785 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T21:57:49,787 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731707899787 2024-11-15T21:57:49,787 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-15T21:57:49,797 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=229a036fd38ae13e7359b15bc2f2e587, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=2 2024-11-15T21:57:49,797 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dd723478fcfc0e25b606965c9135244b, regionState=OPEN, lastHost=ae3311ea5ccb,32871,1731707679650, regionLocation=ae3311ea5ccb,32871,1731707679650, openSeqNum=2 2024-11-15T21:57:49,797 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50df10b3ae47c5dc7c4521c4c58c192f, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=2 2024-11-15T21:57:49,797 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4256fff2fa33299a6264dd460a4003fb, regionState=OPEN, lastHost=ae3311ea5ccb,32871,1731707679650, regionLocation=ae3311ea5ccb,32871,1731707679650, openSeqNum=2 2024-11-15T21:57:49,798 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2c83200b2c422973a68afce1173aae30, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=2 2024-11-15T21:57:49,798 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f13d1331edcf64c426c3bf9433330e4, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=2 2024-11-15T21:57:49,798 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9ef867a298d175eca9d85304f2faa69, regionState=OPEN, lastHost=ae3311ea5ccb,32871,1731707679650, regionLocation=ae3311ea5ccb,32871,1731707679650, openSeqNum=2 2024-11-15T21:57:49,798 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0890aeec1f0ca0649e6483a2d56f36b4, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=2 2024-11-15T21:57:49,798 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d28ee8a2ac06d2199c72ac7f884ba470, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=2 2024-11-15T21:57:49,798 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aec2e46cb64898ba2a66825d3732f6dc, regionState=OPEN, lastHost=ae3311ea5ccb,32871,1731707679650, regionLocation=ae3311ea5ccb,32871,1731707679650, openSeqNum=2 2024-11-15T21:57:49,798 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cbb6b5b8341f7ff2ada4ac1931a1a588, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=2 2024-11-15T21:57:49,798 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a3f7be24d811f4d6177c28e08f25af2d, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=2 2024-11-15T21:57:49,798 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e3b749fcbfdadfe59f1d1cf94f5406a, regionState=OPEN, lastHost=ae3311ea5ccb,32871,1731707679650, regionLocation=ae3311ea5ccb,32871,1731707679650, openSeqNum=2 2024-11-15T21:57:49,799 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51af260ba7a9b40a7f788a5c93a416b6, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=2 2024-11-15T21:57:49,799 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ab2d104e772f41ee9c56851adad43f0, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=2 2024-11-15T21:57:49,799 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3cf264afea186a5cc96edd3e68a10d45, regionState=OPEN, lastHost=ae3311ea5ccb,32871,1731707679650, regionLocation=ae3311ea5ccb,32871,1731707679650, openSeqNum=2 2024-11-15T21:57:49,799 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d100add17d87270d000a3c39b49f8227, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=2 2024-11-15T21:57:49,799 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d9989b0341ef96818e811f874dbcf91c, regionState=OPEN, lastHost=ae3311ea5ccb,32871,1731707679650, regionLocation=ae3311ea5ccb,32871,1731707679650, openSeqNum=2 2024-11-15T21:57:49,799 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=386f4af1e738f142f3fbde37773d4d2c, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=2 2024-11-15T21:57:49,799 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d6063c53955b74f03cc36bd578a6bd41, regionState=OPEN, lastHost=ae3311ea5ccb,32871,1731707679650, regionLocation=ae3311ea5ccb,32871,1731707679650, openSeqNum=2 2024-11-15T21:57:49,799 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fac6af87aa0fc9826df0515f766c031e, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=2 2024-11-15T21:57:49,799 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ce283f02f0974e39c98759bb2bc9883, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=2 2024-11-15T21:57:49,799 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6694325f9c862a5a1ee02eabf7861cc3, regionState=OPEN, lastHost=ae3311ea5ccb,32871,1731707679650, regionLocation=ae3311ea5ccb,32871,1731707679650, openSeqNum=2 2024-11-15T21:57:49,800 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46b5f83abe9585829b26884a1b26b43f, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=2 2024-11-15T21:57:49,800 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1031049c0b4149d1189db33e38ab56cd, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=2 2024-11-15T21:57:49,800 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e76ae188febdca91d319999f619d9e11, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=2 2024-11-15T21:57:49,800 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-15T21:57:49,800 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731707929800 2024-11-15T21:57:49,800 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731707989800 2024-11-15T21:57:49,800 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 13 msec 2024-11-15T21:57:49,801 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure ae3311ea5ccb,32871,1731707679650, splitWal=true, meta=true, isMeta: false 2024-11-15T21:57:49,806 INFO [PEWorker-1 {}] master.SplitWALManager(105): ae3311ea5ccb,32871,1731707679650 WAL count=1, meta=false 2024-11-15T21:57:49,806 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure ae3311ea5ccb%2C32871%2C1731707679650.1731707681383}] 2024-11-15T21:57:49,806 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-15T21:57:49,811 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:57:49,812 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure ae3311ea5ccb%2C32871%2C1731707679650.1731707681383, worker=ae3311ea5ccb,37455,1731707679725}] 2024-11-15T21:57:49,971 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48249, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-15T21:57:49,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=63 2024-11-15T21:57:50,001 INFO [RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(299): Splitting hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/WALs/ae3311ea5ccb,32871,1731707679650-splitting/ae3311ea5ccb%2C32871%2C1731707679650.1731707681383, size=327.1 K (334920bytes) 2024-11-15T21:57:50,002 INFO [RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/WALs/ae3311ea5ccb,32871,1731707679650-splitting/ae3311ea5ccb%2C32871%2C1731707679650.1731707681383 2024-11-15T21:57:50,002 INFO [RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/WALs/ae3311ea5ccb,32871,1731707679650-splitting/ae3311ea5ccb%2C32871%2C1731707679650.1731707681383 after 0ms 2024-11-15T21:57:50,029 INFO [RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(310): Open hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/WALs/ae3311ea5ccb,32871,1731707679650-splitting/ae3311ea5ccb%2C32871%2C1731707679650.1731707681383 took 28ms 2024-11-15T21:57:50,048 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 6e3b749fcbfdadfe59f1d1cf94f5406a: last_flushed_sequence_id: 18446744073709551615 2024-11-15T21:57:50,050 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for d6063c53955b74f03cc36bd578a6bd41: last_flushed_sequence_id: 18446744073709551615 2024-11-15T21:57:50,064 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for f9ef867a298d175eca9d85304f2faa69: last_flushed_sequence_id: 18446744073709551615 2024-11-15T21:57:50,075 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 6694325f9c862a5a1ee02eabf7861cc3: last_flushed_sequence_id: 18446744073709551615 2024-11-15T21:57:50,080 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for d9989b0341ef96818e811f874dbcf91c: last_flushed_sequence_id: 18446744073709551615 2024-11-15T21:57:50,082 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 3cf264afea186a5cc96edd3e68a10d45: last_flushed_sequence_id: 18446744073709551615 2024-11-15T21:57:50,089 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 4256fff2fa33299a6264dd460a4003fb: last_flushed_sequence_id: 18446744073709551615 2024-11-15T21:57:50,091 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for aec2e46cb64898ba2a66825d3732f6dc: last_flushed_sequence_id: 18446744073709551615 2024-11-15T21:57:50,092 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for dd723478fcfc0e25b606965c9135244b: last_flushed_sequence_id: 18446744073709551615 2024-11-15T21:57:50,100 INFO [RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0-1-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp 2024-11-15T21:57:50,113 INFO [RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0-1-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dd723478fcfc0e25b606965c9135244b/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp 2024-11-15T21:57:50,124 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/WALs/ae3311ea5ccb,32871,1731707679650-splitting/ae3311ea5ccb%2C32871%2C1731707679650.1731707681383 so closing down 2024-11-15T21:57:50,124 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-15T21:57:50,141 INFO [RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0-1-Writer-1 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp 2024-11-15T21:57:50,153 INFO [RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0-1-Writer-2 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp 2024-11-15T21:57:50,153 INFO [RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0-1-Writer-1 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4256fff2fa33299a6264dd460a4003fb/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp 2024-11-15T21:57:50,165 INFO [RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0-1-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp 2024-11-15T21:57:50,170 INFO [RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0-1-Writer-2 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aec2e46cb64898ba2a66825d3732f6dc/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp 2024-11-15T21:57:50,181 INFO [RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0-1-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3cf264afea186a5cc96edd3e68a10d45/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp 2024-11-15T21:57:50,182 INFO [RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0-1-Writer-1 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp 2024-11-15T21:57:50,187 INFO [RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0-1-Writer-2 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp 2024-11-15T21:57:50,189 INFO [RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0-1-Writer-1 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6694325f9c862a5a1ee02eabf7861cc3/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp 2024-11-15T21:57:50,195 INFO [RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0-1-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp 2024-11-15T21:57:50,202 INFO [RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0-1-Writer-2 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6e3b749fcbfdadfe59f1d1cf94f5406a/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp 2024-11-15T21:57:50,203 INFO [RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0-1-Writer-1 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp 2024-11-15T21:57:50,221 INFO [RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0-1-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d6063c53955b74f03cc36bd578a6bd41/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp 2024-11-15T21:57:50,222 INFO [RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0-1-Writer-1 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d9989b0341ef96818e811f874dbcf91c/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp 2024-11-15T21:57:50,229 INFO [RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0-1-Writer-2 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp 2024-11-15T21:57:50,254 INFO [RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0-1-Writer-2 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f9ef867a298d175eca9d85304f2faa69/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp 2024-11-15T21:57:50,255 INFO [RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(145): 3 split writer threads finished 2024-11-15T21:57:50,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741870_1046 (size=36767) 2024-11-15T21:57:50,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741870_1046 (size=36767) 2024-11-15T21:57:50,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741870_1046 (size=36767) 2024-11-15T21:57:50,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741869_1045 (size=36767) 2024-11-15T21:57:50,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741869_1045 (size=36767) 2024-11-15T21:57:50,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741869_1045 (size=36767) 2024-11-15T21:57:50,365 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6694325f9c862a5a1ee02eabf7861cc3/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-15T21:57:50,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741871_1047 (size=36767) 2024-11-15T21:57:50,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741871_1047 (size=36767) 2024-11-15T21:57:50,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741871_1047 (size=36767) 2024-11-15T21:57:50,379 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6694325f9c862a5a1ee02eabf7861cc3/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp to hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6694325f9c862a5a1ee02eabf7861cc3/recovered.edits/0000000000000000004 2024-11-15T21:57:50,379 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6e3b749fcbfdadfe59f1d1cf94f5406a/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-15T21:57:50,405 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6e3b749fcbfdadfe59f1d1cf94f5406a/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp to hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6e3b749fcbfdadfe59f1d1cf94f5406a/recovered.edits/0000000000000000004 2024-11-15T21:57:50,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741872_1048 (size=36767) 2024-11-15T21:57:50,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741872_1048 (size=36767) 2024-11-15T21:57:50,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741872_1048 (size=36767) 2024-11-15T21:57:50,455 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dd723478fcfc0e25b606965c9135244b/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp (wrote 1 edits, skipped 0 edits in 5 ms) 2024-11-15T21:57:50,469 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dd723478fcfc0e25b606965c9135244b/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp to hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dd723478fcfc0e25b606965c9135244b/recovered.edits/0000000000000000004 2024-11-15T21:57:50,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741873_1049 (size=36767) 2024-11-15T21:57:50,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741873_1049 (size=36767) 2024-11-15T21:57:50,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741873_1049 (size=36767) 2024-11-15T21:57:50,509 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4256fff2fa33299a6264dd460a4003fb/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-15T21:57:50,529 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4256fff2fa33299a6264dd460a4003fb/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp to hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4256fff2fa33299a6264dd460a4003fb/recovered.edits/0000000000000000004 2024-11-15T21:57:50,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741874_1050 (size=36767) 2024-11-15T21:57:50,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741874_1050 (size=36767) 2024-11-15T21:57:50,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741874_1050 (size=36767) 2024-11-15T21:57:50,545 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f9ef867a298d175eca9d85304f2faa69/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-15T21:57:50,565 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f9ef867a298d175eca9d85304f2faa69/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp to hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f9ef867a298d175eca9d85304f2faa69/recovered.edits/0000000000000000004 2024-11-15T21:57:50,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741875_1051 (size=36767) 2024-11-15T21:57:50,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741875_1051 (size=36767) 2024-11-15T21:57:50,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741875_1051 (size=36767) 2024-11-15T21:57:50,610 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d6063c53955b74f03cc36bd578a6bd41/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-15T21:57:50,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741876_1052 (size=36767) 2024-11-15T21:57:50,641 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d6063c53955b74f03cc36bd578a6bd41/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp to hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d6063c53955b74f03cc36bd578a6bd41/recovered.edits/0000000000000000004 2024-11-15T21:57:50,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741876_1052 (size=36767) 2024-11-15T21:57:50,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741876_1052 (size=36767) 2024-11-15T21:57:50,649 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aec2e46cb64898ba2a66825d3732f6dc/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-15T21:57:50,672 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aec2e46cb64898ba2a66825d3732f6dc/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp to hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aec2e46cb64898ba2a66825d3732f6dc/recovered.edits/0000000000000000004 2024-11-15T21:57:50,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741877_1053 (size=36767) 2024-11-15T21:57:50,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741877_1053 (size=36767) 2024-11-15T21:57:50,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741877_1053 (size=36767) 2024-11-15T21:57:50,721 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3cf264afea186a5cc96edd3e68a10d45/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-15T21:57:50,728 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d9989b0341ef96818e811f874dbcf91c/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-15T21:57:50,729 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3cf264afea186a5cc96edd3e68a10d45/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp to hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3cf264afea186a5cc96edd3e68a10d45/recovered.edits/0000000000000000004 2024-11-15T21:57:50,733 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d9989b0341ef96818e811f874dbcf91c/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp to hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d9989b0341ef96818e811f874dbcf91c/recovered.edits/0000000000000000004 2024-11-15T21:57:50,734 INFO [RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(425): Processed 18 edits across 9 Regions in 696 ms; skipped=9; WAL=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/WALs/ae3311ea5ccb,32871,1731707679650-splitting/ae3311ea5ccb%2C32871%2C1731707679650.1731707681383, size=327.1 K, length=334920, corrupted=false, cancelled=false 2024-11-15T21:57:50,734 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(428): Completed split of hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/WALs/ae3311ea5ccb,32871,1731707679650-splitting/ae3311ea5ccb%2C32871%2C1731707679650.1731707681383, journal: Splitting hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/WALs/ae3311ea5ccb,32871,1731707679650-splitting/ae3311ea5ccb%2C32871%2C1731707679650.1731707681383, size=327.1 K (334920bytes) at 1731707870002Creating recovered edits writer path=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dd723478fcfc0e25b606965c9135244b/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp at 1731707870113 (+111 ms)Finishing writing output for hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/WALs/ae3311ea5ccb,32871,1731707679650-splitting/ae3311ea5ccb%2C32871%2C1731707679650.1731707681383 so closing down at 1731707870124 (+11 ms)Creating recovered edits writer path=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4256fff2fa33299a6264dd460a4003fb/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp at 1731707870153 (+29 ms)Creating recovered edits writer path=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aec2e46cb64898ba2a66825d3732f6dc/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp at 1731707870170 (+17 ms)Creating recovered edits writer path=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3cf264afea186a5cc96edd3e68a10d45/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp at 1731707870181 (+11 ms)Creating recovered edits writer path=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6694325f9c862a5a1ee02eabf7861cc3/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp at 1731707870189 (+8 ms)Creating recovered edits writer path=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6e3b749fcbfdadfe59f1d1cf94f5406a/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp at 1731707870202 (+13 ms)Creating recovered edits writer path=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d6063c53955b74f03cc36bd578a6bd41/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp at 1731707870221 (+19 ms)Creating recovered edits writer path=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d9989b0341ef96818e811f874dbcf91c/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp at 1731707870222 (+1 ms)Creating recovered edits writer path=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f9ef867a298d175eca9d85304f2faa69/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp at 1731707870254 (+32 ms)3 split writer threads finished at 1731707870255 (+1 ms)Closed recovered edits writer path=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6694325f9c862a5a1ee02eabf7861cc3/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731707870365 (+110 ms)Rename recovered edits hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6694325f9c862a5a1ee02eabf7861cc3/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp to hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6694325f9c862a5a1ee02eabf7861cc3/recovered.edits/0000000000000000004 at 1731707870379 (+14 ms)Closed recovered edits writer path=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6e3b749fcbfdadfe59f1d1cf94f5406a/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731707870379Rename recovered edits hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6e3b749fcbfdadfe59f1d1cf94f5406a/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp to hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6e3b749fcbfdadfe59f1d1cf94f5406a/recovered.edits/0000000000000000004 at 1731707870405 (+26 ms)Closed recovered edits writer path=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dd723478fcfc0e25b606965c9135244b/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp (wrote 1 edits, skipped 0 edits in 5 ms) at 1731707870455 (+50 ms)Rename recovered edits hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dd723478fcfc0e25b606965c9135244b/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp to hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dd723478fcfc0e25b606965c9135244b/recovered.edits/0000000000000000004 at 1731707870470 (+15 ms)Closed recovered edits writer path=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4256fff2fa33299a6264dd460a4003fb/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731707870509 (+39 ms)Rename recovered edits hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4256fff2fa33299a6264dd460a4003fb/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp to hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4256fff2fa33299a6264dd460a4003fb/recovered.edits/0000000000000000004 at 1731707870529 (+20 ms)Closed recovered edits writer path=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f9ef867a298d175eca9d85304f2faa69/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731707870545 (+16 ms)Rename recovered edits hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f9ef867a298d175eca9d85304f2faa69/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp to hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f9ef867a298d175eca9d85304f2faa69/recovered.edits/0000000000000000004 at 1731707870565 (+20 ms)Closed recovered edits writer path=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d6063c53955b74f03cc36bd578a6bd41/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731707870610 (+45 ms)Rename recovered edits hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d6063c53955b74f03cc36bd578a6bd41/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp to hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d6063c53955b74f03cc36bd578a6bd41/recovered.edits/0000000000000000004 at 1731707870641 (+31 ms)Closed recovered edits writer path=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aec2e46cb64898ba2a66825d3732f6dc/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731707870649 (+8 ms)Rename recovered edits hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aec2e46cb64898ba2a66825d3732f6dc/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp to hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aec2e46cb64898ba2a66825d3732f6dc/recovered.edits/0000000000000000004 at 1731707870672 (+23 ms)Closed recovered edits writer path=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3cf264afea186a5cc96edd3e68a10d45/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731707870721 (+49 ms)Closed recovered edits writer path=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d9989b0341ef96818e811f874dbcf91c/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731707870728 (+7 ms)Rename recovered edits hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3cf264afea186a5cc96edd3e68a10d45/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp to hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3cf264afea186a5cc96edd3e68a10d45/recovered.edits/0000000000000000004 at 1731707870729 (+1 ms)Rename recovered edits hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d9989b0341ef96818e811f874dbcf91c/recovered.edits/0000000000000000004-ae3311ea5ccb%2C32871%2C1731707679650.1731707681383.temp to hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d9989b0341ef96818e811f874dbcf91c/recovered.edits/0000000000000000004 at 1731707870733 (+4 ms)Processed 18 edits across 9 Regions in 696 ms; skipped=9; WAL=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/WALs/ae3311ea5ccb,32871,1731707679650-splitting/ae3311ea5ccb%2C32871%2C1731707679650.1731707681383, size=327.1 K, length=334920, corrupted=false, cancelled=false at 1731707870734 (+1 ms) 2024-11-15T21:57:50,734 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/WALs/ae3311ea5ccb,32871,1731707679650-splitting/ae3311ea5ccb%2C32871%2C1731707679650.1731707681383 2024-11-15T21:57:50,734 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-15T21:57:50,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36531 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-15T21:57:50,742 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/WALs/ae3311ea5ccb,32871,1731707679650-splitting/ae3311ea5ccb%2C32871%2C1731707679650.1731707681383 to hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/oldWALs 2024-11-15T21:57:50,762 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-15T21:57:50,762 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ae3311ea5ccb%2C32871%2C1731707679650.1731707681383, worker=ae3311ea5ccb,37455,1731707679725 in 930 msec 2024-11-15T21:57:50,766 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:57:50,777 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=57 2024-11-15T21:57:50,777 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ae3311ea5ccb%2C32871%2C1731707679650.1731707681383, worker=ae3311ea5ccb,37455,1731707679725 in 960 msec 2024-11-15T21:57:50,788 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/WALs/ae3311ea5ccb,32871,1731707679650-splitting dir is empty, no logs to split. 2024-11-15T21:57:50,788 INFO [PEWorker-1 {}] master.SplitWALManager(105): ae3311ea5ccb,32871,1731707679650 WAL count=0, meta=false 2024-11-15T21:57:50,788 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if ae3311ea5ccb,32871,1731707679650 WAL splitting is done? wals=0, meta=false 2024-11-15T21:57:50,789 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-15T21:57:50,789 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN, hasLock=true; ServerCrashProcedure ae3311ea5ccb,32871,1731707679650, splitWal=true, meta=true 2024-11-15T21:57:50,789 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T21:57:50,790 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN, hasLock=true; ServerCrashProcedure ae3311ea5ccb,32871,1731707679650, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T21:57:50,805 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 6 exec state=pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN, hasLock=true; ServerCrashProcedure ae3311ea5ccb,32871,1731707679650, splitWal=true, meta=true 2024-11-15T21:57:50,806 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T21:57:50,806 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T21:57:50,806 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T21:57:50,809 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T21:57:50,812 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T21:57:50,812 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T21:57:50,812 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T21:57:50,813 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T21:57:50,822 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 9 msec 2024-11-15T21:57:50,824 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T21:57:50,824 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T21:57:50,824 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T21:57:50,824 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T21:57:50,824 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T21:57:50,824 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T21:57:50,824 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T21:57:50,824 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T21:57:50,825 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-15T21:57:50,826 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_DELETE_SPLIT_WALS_DIR, hasLock=false; ServerCrashProcedure ae3311ea5ccb,32871,1731707679650, splitWal=true, meta=true 2024-11-15T21:57:50,826 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ae3311ea5ccb%2C32871%2C1731707679650.meta.1731707681806.meta, worker=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:57:50,826 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ae3311ea5ccb%2C32871%2C1731707679650.meta.1731707681806.meta, worker=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:57:50,826 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T21:57:50,826 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ae3311ea5ccb,40941,1731707679516 2024-11-15T21:57:50,826 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ae3311ea5ccb%2C32871%2C1731707679650.1731707681383, worker=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:57:50,826 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ae3311ea5ccb%2C32871%2C1731707679650.1731707681383, worker=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:57:50,826 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_DELETE_SPLIT_WALS_DIR, hasLock=false; ServerCrashProcedure ae3311ea5ccb,32871,1731707679650, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T21:57:50,826 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ae3311ea5ccb%2C32871%2C1731707679650.meta.1731707681806.meta, worker=ae3311ea5ccb,37455,1731707679725 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T21:57:50,826 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ae3311ea5ccb%2C32871%2C1731707679650.meta.1731707681806.meta, worker=ae3311ea5ccb,37455,1731707679725 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T21:57:50,826 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T21:57:50,826 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ae3311ea5ccb,40941,1731707679516 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T21:57:50,826 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ae3311ea5ccb%2C32871%2C1731707679650.1731707681383, worker=ae3311ea5ccb,37455,1731707679725 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T21:57:50,826 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ae3311ea5ccb%2C32871%2C1731707679650.1731707681383, worker=ae3311ea5ccb,37455,1731707679725 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T21:57:50,826 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-15T21:57:50,830 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=61 2024-11-15T21:57:50,830 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ae3311ea5ccb,40941,1731707679516, table=hbase:meta, region=1588230740 2024-11-15T21:57:50,830 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ae3311ea5ccb,40941,1731707679516, state=OPEN 2024-11-15T21:57:50,841 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37455-0x100f86521fe0003, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T21:57:50,841 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36531-0x100f86521fe0000, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T21:57:50,842 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T21:57:50,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40941-0x100f86521fe0001, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T21:57:50,844 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T21:57:50,845 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T21:57:50,849 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731707900849 2024-11-15T21:57:50,849 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-15T21:57:50,857 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=229a036fd38ae13e7359b15bc2f2e587, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=2 2024-11-15T21:57:50,857 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dd723478fcfc0e25b606965c9135244b, regionState=OPEN, lastHost=ae3311ea5ccb,32871,1731707679650, regionLocation=ae3311ea5ccb,32871,1731707679650, openSeqNum=2 2024-11-15T21:57:50,857 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50df10b3ae47c5dc7c4521c4c58c192f, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=2 2024-11-15T21:57:50,857 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4256fff2fa33299a6264dd460a4003fb, regionState=OPEN, lastHost=ae3311ea5ccb,32871,1731707679650, regionLocation=ae3311ea5ccb,32871,1731707679650, openSeqNum=2 2024-11-15T21:57:50,857 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2c83200b2c422973a68afce1173aae30, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=2 2024-11-15T21:57:50,857 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f13d1331edcf64c426c3bf9433330e4, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=2 2024-11-15T21:57:50,857 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9ef867a298d175eca9d85304f2faa69, regionState=OPEN, lastHost=ae3311ea5ccb,32871,1731707679650, regionLocation=ae3311ea5ccb,32871,1731707679650, openSeqNum=2 2024-11-15T21:57:50,858 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0890aeec1f0ca0649e6483a2d56f36b4, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=2 2024-11-15T21:57:50,858 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d28ee8a2ac06d2199c72ac7f884ba470, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=2 2024-11-15T21:57:50,858 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aec2e46cb64898ba2a66825d3732f6dc, regionState=OPEN, lastHost=ae3311ea5ccb,32871,1731707679650, regionLocation=ae3311ea5ccb,32871,1731707679650, openSeqNum=2 2024-11-15T21:57:50,858 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cbb6b5b8341f7ff2ada4ac1931a1a588, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=2 2024-11-15T21:57:50,858 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a3f7be24d811f4d6177c28e08f25af2d, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=2 2024-11-15T21:57:50,858 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e3b749fcbfdadfe59f1d1cf94f5406a, regionState=OPEN, lastHost=ae3311ea5ccb,32871,1731707679650, regionLocation=ae3311ea5ccb,32871,1731707679650, openSeqNum=2 2024-11-15T21:57:50,858 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51af260ba7a9b40a7f788a5c93a416b6, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=2 2024-11-15T21:57:50,858 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ab2d104e772f41ee9c56851adad43f0, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=2 2024-11-15T21:57:50,858 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3cf264afea186a5cc96edd3e68a10d45, regionState=OPEN, lastHost=ae3311ea5ccb,32871,1731707679650, regionLocation=ae3311ea5ccb,32871,1731707679650, openSeqNum=2 2024-11-15T21:57:50,858 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d100add17d87270d000a3c39b49f8227, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=2 2024-11-15T21:57:50,858 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d9989b0341ef96818e811f874dbcf91c, regionState=OPEN, lastHost=ae3311ea5ccb,32871,1731707679650, regionLocation=ae3311ea5ccb,32871,1731707679650, openSeqNum=2 2024-11-15T21:57:50,859 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=386f4af1e738f142f3fbde37773d4d2c, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=2 2024-11-15T21:57:50,859 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d6063c53955b74f03cc36bd578a6bd41, regionState=OPEN, lastHost=ae3311ea5ccb,32871,1731707679650, regionLocation=ae3311ea5ccb,32871,1731707679650, openSeqNum=2 2024-11-15T21:57:50,859 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fac6af87aa0fc9826df0515f766c031e, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=2 2024-11-15T21:57:50,859 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ce283f02f0974e39c98759bb2bc9883, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=2 2024-11-15T21:57:50,859 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6694325f9c862a5a1ee02eabf7861cc3, regionState=OPEN, lastHost=ae3311ea5ccb,32871,1731707679650, regionLocation=ae3311ea5ccb,32871,1731707679650, openSeqNum=2 2024-11-15T21:57:50,859 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46b5f83abe9585829b26884a1b26b43f, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=2 2024-11-15T21:57:50,859 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1031049c0b4149d1189db33e38ab56cd, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=2 2024-11-15T21:57:50,859 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e76ae188febdca91d319999f619d9e11, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=2 2024-11-15T21:57:50,859 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-15T21:57:50,859 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731707930859 2024-11-15T21:57:50,859 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731707990859 2024-11-15T21:57:50,859 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-15T21:57:50,882 INFO [PEWorker-1 {}] master.MasterWalManager(331): Log dir for server ae3311ea5ccb,32871,1731707679650 does not exist 2024-11-15T21:57:50,882 INFO [PEWorker-1 {}] master.SplitWALManager(105): ae3311ea5ccb,32871,1731707679650 WAL count=0, meta=false 2024-11-15T21:57:50,882 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if ae3311ea5ccb,32871,1731707679650 WAL splitting is done? wals=0, meta=false 2024-11-15T21:57:50,893 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for ae3311ea5ccb,32871,1731707679650 failed, ignore...File hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/WALs/ae3311ea5ccb,32871,1731707679650-splitting does not exist. 2024-11-15T21:57:50,893 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-15T21:57:50,901 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=dd723478fcfc0e25b606965c9135244b, ASSIGN}, {pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=aec2e46cb64898ba2a66825d3732f6dc, ASSIGN}, {pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=f9ef867a298d175eca9d85304f2faa69, ASSIGN}, {pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3cf264afea186a5cc96edd3e68a10d45, ASSIGN}, {pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4256fff2fa33299a6264dd460a4003fb, ASSIGN}, {pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d9989b0341ef96818e811f874dbcf91c, ASSIGN}, {pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=6e3b749fcbfdadfe59f1d1cf94f5406a, ASSIGN}, {pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d6063c53955b74f03cc36bd578a6bd41, ASSIGN}, {pid=72, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=6694325f9c862a5a1ee02eabf7861cc3, ASSIGN}] 2024-11-15T21:57:50,901 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-15T21:57:50,901 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_FINISH, hasLock=true; ServerCrashProcedure ae3311ea5ccb,32871,1731707679650, splitWal=true, meta=true 2024-11-15T21:57:50,901 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T21:57:50,901 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_FINISH, hasLock=true; ServerCrashProcedure ae3311ea5ccb,32871,1731707679650, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T21:57:51,110 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 7 exec state=pid=57, state=WAITING:SERVER_CRASH_FINISH, hasLock=true; ServerCrashProcedure ae3311ea5ccb,32871,1731707679650, splitWal=true, meta=true 2024-11-15T21:57:51,110 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T21:57:51,110 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T21:57:51,110 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T21:57:51,110 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T21:57:51,110 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T21:57:51,110 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T21:57:51,110 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T21:57:51,111 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T21:57:51,111 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T21:57:51,112 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T21:57:51,113 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T21:57:51,113 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T21:57:51,113 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T21:57:51,113 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T21:57:51,113 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T21:57:51,113 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T21:57:51,113 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T21:57:51,114 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-15T21:57:51,115 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN, hasLock=false; ServerCrashProcedure ae3311ea5ccb,32871,1731707679650, splitWal=true, meta=true 2024-11-15T21:57:51,115 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ae3311ea5ccb%2C32871%2C1731707679650.meta.1731707681806.meta, worker=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:57:51,115 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ae3311ea5ccb%2C32871%2C1731707679650.meta.1731707681806.meta, worker=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:57:51,115 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T21:57:51,116 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ae3311ea5ccb,40941,1731707679516 2024-11-15T21:57:51,116 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ae3311ea5ccb%2C32871%2C1731707679650.1731707681383, worker=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:57:51,116 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ae3311ea5ccb%2C32871%2C1731707679650.1731707681383, worker=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:57:51,116 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN, hasLock=false; ServerCrashProcedure ae3311ea5ccb,32871,1731707679650, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T21:57:51,116 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ae3311ea5ccb%2C32871%2C1731707679650.meta.1731707681806.meta, worker=ae3311ea5ccb,37455,1731707679725 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T21:57:51,116 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ae3311ea5ccb%2C32871%2C1731707679650.meta.1731707681806.meta, worker=ae3311ea5ccb,37455,1731707679725 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T21:57:51,116 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T21:57:51,116 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ae3311ea5ccb,40941,1731707679516 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T21:57:51,116 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ae3311ea5ccb%2C32871%2C1731707679650.1731707681383, worker=ae3311ea5ccb,37455,1731707679725 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T21:57:51,116 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ae3311ea5ccb%2C32871%2C1731707679650.1731707681383, worker=ae3311ea5ccb,37455,1731707679725 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T21:57:51,116 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-15T21:57:51,116 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=61 2024-11-15T21:57:51,117 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ae3311ea5ccb,40941,1731707679516, table=hbase:meta, region=1588230740 2024-11-15T21:57:51,117 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ae3311ea5ccb,40941,1731707679516, state=OPEN 2024-11-15T21:57:51,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40941-0x100f86521fe0001, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T21:57:51,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37455-0x100f86521fe0003, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T21:57:51,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36531-0x100f86521fe0000, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T21:57:51,119 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T21:57:51,119 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T21:57:51,119 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T21:57:51,120 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731707901120 2024-11-15T21:57:51,120 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-15T21:57:51,129 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=229a036fd38ae13e7359b15bc2f2e587, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=2 2024-11-15T21:57:51,129 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dd723478fcfc0e25b606965c9135244b, regionState=OPEN, lastHost=ae3311ea5ccb,32871,1731707679650, regionLocation=ae3311ea5ccb,32871,1731707679650, openSeqNum=2 2024-11-15T21:57:51,129 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50df10b3ae47c5dc7c4521c4c58c192f, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=2 2024-11-15T21:57:51,130 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4256fff2fa33299a6264dd460a4003fb, regionState=OPEN, lastHost=ae3311ea5ccb,32871,1731707679650, regionLocation=ae3311ea5ccb,32871,1731707679650, openSeqNum=2 2024-11-15T21:57:51,130 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2c83200b2c422973a68afce1173aae30, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=2 2024-11-15T21:57:51,130 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f13d1331edcf64c426c3bf9433330e4, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=2 2024-11-15T21:57:51,130 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9ef867a298d175eca9d85304f2faa69, regionState=OPEN, lastHost=ae3311ea5ccb,32871,1731707679650, regionLocation=ae3311ea5ccb,32871,1731707679650, openSeqNum=2 2024-11-15T21:57:51,130 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0890aeec1f0ca0649e6483a2d56f36b4, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=2 2024-11-15T21:57:51,130 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d28ee8a2ac06d2199c72ac7f884ba470, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=2 2024-11-15T21:57:51,130 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aec2e46cb64898ba2a66825d3732f6dc, regionState=OPEN, lastHost=ae3311ea5ccb,32871,1731707679650, regionLocation=ae3311ea5ccb,32871,1731707679650, openSeqNum=2 2024-11-15T21:57:51,130 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cbb6b5b8341f7ff2ada4ac1931a1a588, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=2 2024-11-15T21:57:51,130 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a3f7be24d811f4d6177c28e08f25af2d, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=2 2024-11-15T21:57:51,130 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e3b749fcbfdadfe59f1d1cf94f5406a, regionState=OPEN, lastHost=ae3311ea5ccb,32871,1731707679650, regionLocation=ae3311ea5ccb,32871,1731707679650, openSeqNum=2 2024-11-15T21:57:51,130 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51af260ba7a9b40a7f788a5c93a416b6, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=2 2024-11-15T21:57:51,131 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ab2d104e772f41ee9c56851adad43f0, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=2 2024-11-15T21:57:51,131 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3cf264afea186a5cc96edd3e68a10d45, regionState=OPEN, lastHost=ae3311ea5ccb,32871,1731707679650, regionLocation=ae3311ea5ccb,32871,1731707679650, openSeqNum=2 2024-11-15T21:57:51,131 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d100add17d87270d000a3c39b49f8227, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=2 2024-11-15T21:57:51,131 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d9989b0341ef96818e811f874dbcf91c, regionState=OPEN, lastHost=ae3311ea5ccb,32871,1731707679650, regionLocation=ae3311ea5ccb,32871,1731707679650, openSeqNum=2 2024-11-15T21:57:51,131 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=386f4af1e738f142f3fbde37773d4d2c, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=2 2024-11-15T21:57:51,131 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d6063c53955b74f03cc36bd578a6bd41, regionState=OPEN, lastHost=ae3311ea5ccb,32871,1731707679650, regionLocation=ae3311ea5ccb,32871,1731707679650, openSeqNum=2 2024-11-15T21:57:51,131 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fac6af87aa0fc9826df0515f766c031e, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=2 2024-11-15T21:57:51,131 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ce283f02f0974e39c98759bb2bc9883, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=2 2024-11-15T21:57:51,131 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6694325f9c862a5a1ee02eabf7861cc3, regionState=OPEN, lastHost=ae3311ea5ccb,32871,1731707679650, regionLocation=ae3311ea5ccb,32871,1731707679650, openSeqNum=2 2024-11-15T21:57:51,131 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46b5f83abe9585829b26884a1b26b43f, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=2 2024-11-15T21:57:51,132 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1031049c0b4149d1189db33e38ab56cd, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=2 2024-11-15T21:57:51,132 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e76ae188febdca91d319999f619d9e11, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=2 2024-11-15T21:57:51,132 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-15T21:57:51,132 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731707931132 2024-11-15T21:57:51,132 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731707991132 2024-11-15T21:57:51,132 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 11 msec 2024-11-15T21:57:51,132 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=dd723478fcfc0e25b606965c9135244b, ASSIGN}, {pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=aec2e46cb64898ba2a66825d3732f6dc, ASSIGN}, {pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=f9ef867a298d175eca9d85304f2faa69, ASSIGN}, {pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3cf264afea186a5cc96edd3e68a10d45, ASSIGN}, {pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4256fff2fa33299a6264dd460a4003fb, ASSIGN}, {pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d9989b0341ef96818e811f874dbcf91c, ASSIGN}, {pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=6e3b749fcbfdadfe59f1d1cf94f5406a, ASSIGN}, {pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d6063c53955b74f03cc36bd578a6bd41, ASSIGN}, {pid=72, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=6694325f9c862a5a1ee02eabf7861cc3, ASSIGN}] 2024-11-15T21:57:51,132 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-15T21:57:51,135 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=6694325f9c862a5a1ee02eabf7861cc3, ASSIGN 2024-11-15T21:57:51,136 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=72, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=6694325f9c862a5a1ee02eabf7861cc3, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-15T21:57:51,137 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d6063c53955b74f03cc36bd578a6bd41, ASSIGN 2024-11-15T21:57:51,138 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d6063c53955b74f03cc36bd578a6bd41, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-15T21:57:51,141 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=6e3b749fcbfdadfe59f1d1cf94f5406a, ASSIGN 2024-11-15T21:57:51,142 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=6e3b749fcbfdadfe59f1d1cf94f5406a, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-15T21:57:51,144 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d9989b0341ef96818e811f874dbcf91c, ASSIGN 2024-11-15T21:57:51,145 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d9989b0341ef96818e811f874dbcf91c, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-15T21:57:51,146 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4256fff2fa33299a6264dd460a4003fb, ASSIGN 2024-11-15T21:57:51,146 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4256fff2fa33299a6264dd460a4003fb, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-15T21:57:51,147 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3cf264afea186a5cc96edd3e68a10d45, ASSIGN 2024-11-15T21:57:51,148 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3cf264afea186a5cc96edd3e68a10d45, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-15T21:57:51,148 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=f9ef867a298d175eca9d85304f2faa69, ASSIGN 2024-11-15T21:57:51,149 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=f9ef867a298d175eca9d85304f2faa69, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-15T21:57:51,150 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=aec2e46cb64898ba2a66825d3732f6dc, ASSIGN 2024-11-15T21:57:51,151 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=aec2e46cb64898ba2a66825d3732f6dc, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-15T21:57:51,151 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=dd723478fcfc0e25b606965c9135244b, ASSIGN 2024-11-15T21:57:51,152 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=dd723478fcfc0e25b606965c9135244b, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-15T21:57:51,287 DEBUG [ae3311ea5ccb:36531 {}] balancer.BalancerClusterState(204): Hosts are {ae3311ea5ccb=0} racks are {/default-rack=0} 2024-11-15T21:57:51,287 DEBUG [ae3311ea5ccb:36531 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-15T21:57:51,287 DEBUG [ae3311ea5ccb:36531 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-15T21:57:51,287 DEBUG [ae3311ea5ccb:36531 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-15T21:57:51,287 DEBUG [ae3311ea5ccb:36531 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-15T21:57:51,287 INFO [ae3311ea5ccb:36531 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-15T21:57:51,287 INFO [ae3311ea5ccb:36531 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-15T21:57:51,287 DEBUG [ae3311ea5ccb:36531 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-15T21:57:51,288 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=d6063c53955b74f03cc36bd578a6bd41, regionState=OPENING, regionLocation=ae3311ea5ccb,40941,1731707679516 2024-11-15T21:57:51,289 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=3cf264afea186a5cc96edd3e68a10d45, regionState=OPENING, regionLocation=ae3311ea5ccb,40941,1731707679516 2024-11-15T21:57:51,289 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=65 updating hbase:meta row=aec2e46cb64898ba2a66825d3732f6dc, regionState=OPENING, regionLocation=ae3311ea5ccb,40941,1731707679516 2024-11-15T21:57:51,290 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=4256fff2fa33299a6264dd460a4003fb, regionState=OPENING, regionLocation=ae3311ea5ccb,40941,1731707679516 2024-11-15T21:57:51,291 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=6694325f9c862a5a1ee02eabf7861cc3, regionState=OPENING, regionLocation=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:57:51,291 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=d9989b0341ef96818e811f874dbcf91c, regionState=OPENING, regionLocation=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:57:51,292 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=6e3b749fcbfdadfe59f1d1cf94f5406a, regionState=OPENING, regionLocation=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:57:51,292 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=f9ef867a298d175eca9d85304f2faa69, regionState=OPENING, regionLocation=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:57:51,293 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=dd723478fcfc0e25b606965c9135244b, regionState=OPENING, regionLocation=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:57:51,294 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d6063c53955b74f03cc36bd578a6bd41, ASSIGN because future has completed 2024-11-15T21:57:51,295 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3cf264afea186a5cc96edd3e68a10d45, ASSIGN because future has completed 2024-11-15T21:57:51,296 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3cf264afea186a5cc96edd3e68a10d45, server=ae3311ea5ccb,40941,1731707679516}] 2024-11-15T21:57:51,297 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=aec2e46cb64898ba2a66825d3732f6dc, ASSIGN because future has completed 2024-11-15T21:57:51,298 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4256fff2fa33299a6264dd460a4003fb, ASSIGN because future has completed 2024-11-15T21:57:51,302 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=72, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=6694325f9c862a5a1ee02eabf7861cc3, ASSIGN because future has completed 2024-11-15T21:57:51,303 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d9989b0341ef96818e811f874dbcf91c, ASSIGN because future has completed 2024-11-15T21:57:51,308 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=6e3b749fcbfdadfe59f1d1cf94f5406a, ASSIGN because future has completed 2024-11-15T21:57:51,310 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6e3b749fcbfdadfe59f1d1cf94f5406a, server=ae3311ea5ccb,37455,1731707679725}] 2024-11-15T21:57:51,310 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=f9ef867a298d175eca9d85304f2faa69, ASSIGN because future has completed 2024-11-15T21:57:51,314 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=dd723478fcfc0e25b606965c9135244b, ASSIGN because future has completed 2024-11-15T21:57:51,316 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure dd723478fcfc0e25b606965c9135244b, server=ae3311ea5ccb,37455,1731707679725}] 2024-11-15T21:57:51,325 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure f9ef867a298d175eca9d85304f2faa69, server=ae3311ea5ccb,37455,1731707679725}] 2024-11-15T21:57:51,329 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure d9989b0341ef96818e811f874dbcf91c, server=ae3311ea5ccb,37455,1731707679725}] 2024-11-15T21:57:51,332 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6694325f9c862a5a1ee02eabf7861cc3, server=ae3311ea5ccb,37455,1731707679725}] 2024-11-15T21:57:51,335 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4256fff2fa33299a6264dd460a4003fb, server=ae3311ea5ccb,40941,1731707679516}] 2024-11-15T21:57:51,338 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure d6063c53955b74f03cc36bd578a6bd41, server=ae3311ea5ccb,40941,1731707679516}] 2024-11-15T21:57:51,341 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure aec2e46cb64898ba2a66825d3732f6dc, server=ae3311ea5ccb,40941,1731707679516}] 2024-11-15T21:57:51,475 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731707682333.aec2e46cb64898ba2a66825d3732f6dc. 2024-11-15T21:57:51,475 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7752): Opening region: {ENCODED => aec2e46cb64898ba2a66825d3732f6dc, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731707682333.aec2e46cb64898ba2a66825d3732f6dc.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-15T21:57:51,475 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true aec2e46cb64898ba2a66825d3732f6dc 2024-11-15T21:57:51,475 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731707682333.aec2e46cb64898ba2a66825d3732f6dc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:57:51,476 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7794): checking encryption for aec2e46cb64898ba2a66825d3732f6dc 2024-11-15T21:57:51,476 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7797): checking classloading for aec2e46cb64898ba2a66825d3732f6dc 2024-11-15T21:57:51,479 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731707682333.6e3b749fcbfdadfe59f1d1cf94f5406a. 2024-11-15T21:57:51,480 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7752): Opening region: {ENCODED => 6e3b749fcbfdadfe59f1d1cf94f5406a, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731707682333.6e3b749fcbfdadfe59f1d1cf94f5406a.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-15T21:57:51,480 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 6e3b749fcbfdadfe59f1d1cf94f5406a 2024-11-15T21:57:51,480 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731707682333.6e3b749fcbfdadfe59f1d1cf94f5406a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:57:51,480 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7794): checking encryption for 6e3b749fcbfdadfe59f1d1cf94f5406a 2024-11-15T21:57:51,480 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7797): checking classloading for 6e3b749fcbfdadfe59f1d1cf94f5406a 2024-11-15T21:57:51,482 INFO [StoreOpener-aec2e46cb64898ba2a66825d3732f6dc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region aec2e46cb64898ba2a66825d3732f6dc 2024-11-15T21:57:51,482 INFO [StoreOpener-6e3b749fcbfdadfe59f1d1cf94f5406a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 6e3b749fcbfdadfe59f1d1cf94f5406a 2024-11-15T21:57:51,483 INFO [StoreOpener-aec2e46cb64898ba2a66825d3732f6dc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aec2e46cb64898ba2a66825d3732f6dc columnFamilyName colfamily11 2024-11-15T21:57:51,483 DEBUG [StoreOpener-aec2e46cb64898ba2a66825d3732f6dc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:57:51,483 INFO [StoreOpener-6e3b749fcbfdadfe59f1d1cf94f5406a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6e3b749fcbfdadfe59f1d1cf94f5406a columnFamilyName colfamily11 2024-11-15T21:57:51,483 DEBUG [StoreOpener-6e3b749fcbfdadfe59f1d1cf94f5406a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:57:51,484 INFO [StoreOpener-6e3b749fcbfdadfe59f1d1cf94f5406a-1 {}] regionserver.HStore(327): Store=6e3b749fcbfdadfe59f1d1cf94f5406a/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:57:51,484 INFO [StoreOpener-6e3b749fcbfdadfe59f1d1cf94f5406a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 6e3b749fcbfdadfe59f1d1cf94f5406a 2024-11-15T21:57:51,485 INFO [StoreOpener-aec2e46cb64898ba2a66825d3732f6dc-1 {}] regionserver.HStore(327): Store=aec2e46cb64898ba2a66825d3732f6dc/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:57:51,485 INFO [StoreOpener-aec2e46cb64898ba2a66825d3732f6dc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region aec2e46cb64898ba2a66825d3732f6dc 2024-11-15T21:57:51,486 INFO [StoreOpener-aec2e46cb64898ba2a66825d3732f6dc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aec2e46cb64898ba2a66825d3732f6dc columnFamilyName colfamily21 2024-11-15T21:57:51,486 DEBUG [StoreOpener-aec2e46cb64898ba2a66825d3732f6dc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:57:51,487 INFO [StoreOpener-aec2e46cb64898ba2a66825d3732f6dc-1 {}] regionserver.HStore(327): Store=aec2e46cb64898ba2a66825d3732f6dc/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:57:51,487 INFO [StoreOpener-aec2e46cb64898ba2a66825d3732f6dc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region aec2e46cb64898ba2a66825d3732f6dc 2024-11-15T21:57:51,487 INFO [StoreOpener-aec2e46cb64898ba2a66825d3732f6dc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aec2e46cb64898ba2a66825d3732f6dc columnFamilyName colfamily31 2024-11-15T21:57:51,487 DEBUG [StoreOpener-aec2e46cb64898ba2a66825d3732f6dc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:57:51,488 INFO [StoreOpener-aec2e46cb64898ba2a66825d3732f6dc-1 {}] regionserver.HStore(327): Store=aec2e46cb64898ba2a66825d3732f6dc/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:57:51,488 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1038): replaying wal for aec2e46cb64898ba2a66825d3732f6dc 2024-11-15T21:57:51,489 INFO [StoreOpener-6e3b749fcbfdadfe59f1d1cf94f5406a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6e3b749fcbfdadfe59f1d1cf94f5406a columnFamilyName colfamily21 2024-11-15T21:57:51,489 DEBUG [StoreOpener-6e3b749fcbfdadfe59f1d1cf94f5406a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:57:51,491 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aec2e46cb64898ba2a66825d3732f6dc 2024-11-15T21:57:51,497 INFO [StoreOpener-6e3b749fcbfdadfe59f1d1cf94f5406a-1 {}] regionserver.HStore(327): Store=6e3b749fcbfdadfe59f1d1cf94f5406a/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:57:51,497 INFO [StoreOpener-6e3b749fcbfdadfe59f1d1cf94f5406a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 6e3b749fcbfdadfe59f1d1cf94f5406a 2024-11-15T21:57:51,502 INFO [StoreOpener-6e3b749fcbfdadfe59f1d1cf94f5406a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6e3b749fcbfdadfe59f1d1cf94f5406a columnFamilyName colfamily31 2024-11-15T21:57:51,502 DEBUG [StoreOpener-6e3b749fcbfdadfe59f1d1cf94f5406a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:57:51,504 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aec2e46cb64898ba2a66825d3732f6dc 2024-11-15T21:57:51,505 INFO [StoreOpener-6e3b749fcbfdadfe59f1d1cf94f5406a-1 {}] regionserver.HStore(327): Store=6e3b749fcbfdadfe59f1d1cf94f5406a/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:57:51,505 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1038): replaying wal for 6e3b749fcbfdadfe59f1d1cf94f5406a 2024-11-15T21:57:51,507 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aec2e46cb64898ba2a66825d3732f6dc/recovered.edits/0000000000000000004 2024-11-15T21:57:51,510 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6e3b749fcbfdadfe59f1d1cf94f5406a 2024-11-15T21:57:51,519 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aec2e46cb64898ba2a66825d3732f6dc/recovered.edits/0000000000000000004 2024-11-15T21:57:51,519 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(2902): Flushing aec2e46cb64898ba2a66825d3732f6dc 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-15T21:57:51,525 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6e3b749fcbfdadfe59f1d1cf94f5406a 2024-11-15T21:57:51,532 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6e3b749fcbfdadfe59f1d1cf94f5406a/recovered.edits/0000000000000000004 2024-11-15T21:57:51,548 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6e3b749fcbfdadfe59f1d1cf94f5406a/recovered.edits/0000000000000000004 2024-11-15T21:57:51,548 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(2902): Flushing 6e3b749fcbfdadfe59f1d1cf94f5406a 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-15T21:57:51,622 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aec2e46cb64898ba2a66825d3732f6dc/.tmp/colfamily11/7813970e1cce4d0e8921a37df1604ceb is 52, key is jjj/colfamily11:colfamily11/1731707686586/Put/seqid=0 2024-11-15T21:57:51,623 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6e3b749fcbfdadfe59f1d1cf94f5406a/.tmp/colfamily11/886a45eb14a94989a020391fa00c698b is 52, key is mmm/colfamily11:colfamily11/1731707686652/Put/seqid=0 2024-11-15T21:57:51,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741878_1054 (size=39263) 2024-11-15T21:57:51,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741878_1054 (size=39263) 2024-11-15T21:57:51,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741878_1054 (size=39263) 2024-11-15T21:57:51,704 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aec2e46cb64898ba2a66825d3732f6dc/.tmp/colfamily11/7813970e1cce4d0e8921a37df1604ceb 2024-11-15T21:57:51,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741879_1055 (size=39263) 2024-11-15T21:57:51,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741879_1055 (size=39263) 2024-11-15T21:57:51,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741879_1055 (size=39263) 2024-11-15T21:57:51,724 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aec2e46cb64898ba2a66825d3732f6dc/.tmp/colfamily11/7813970e1cce4d0e8921a37df1604ceb as hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aec2e46cb64898ba2a66825d3732f6dc/colfamily11/7813970e1cce4d0e8921a37df1604ceb 2024-11-15T21:57:51,730 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6e3b749fcbfdadfe59f1d1cf94f5406a/.tmp/colfamily11/886a45eb14a94989a020391fa00c698b 2024-11-15T21:57:51,740 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aec2e46cb64898ba2a66825d3732f6dc/colfamily11/7813970e1cce4d0e8921a37df1604ceb, entries=703, sequenceid=4, filesize=38.3 K 2024-11-15T21:57:51,741 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for aec2e46cb64898ba2a66825d3732f6dc in 221ms, sequenceid=4, compaction requested=false; wal=null 2024-11-15T21:57:51,742 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aec2e46cb64898ba2a66825d3732f6dc/recovered.edits/0000000000000000004 2024-11-15T21:57:51,742 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6e3b749fcbfdadfe59f1d1cf94f5406a/.tmp/colfamily11/886a45eb14a94989a020391fa00c698b as hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6e3b749fcbfdadfe59f1d1cf94f5406a/colfamily11/886a45eb14a94989a020391fa00c698b 2024-11-15T21:57:51,743 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1048): stopping wal replay for aec2e46cb64898ba2a66825d3732f6dc 2024-11-15T21:57:51,743 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1060): Cleaning up temporary data for aec2e46cb64898ba2a66825d3732f6dc 2024-11-15T21:57:51,744 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-15T21:57:51,748 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1093): writing seq id for aec2e46cb64898ba2a66825d3732f6dc 2024-11-15T21:57:51,752 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aec2e46cb64898ba2a66825d3732f6dc/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T21:57:51,758 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6e3b749fcbfdadfe59f1d1cf94f5406a/colfamily11/886a45eb14a94989a020391fa00c698b, entries=703, sequenceid=4, filesize=38.3 K 2024-11-15T21:57:51,758 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 6e3b749fcbfdadfe59f1d1cf94f5406a in 210ms, sequenceid=4, compaction requested=false; wal=null 2024-11-15T21:57:51,762 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6e3b749fcbfdadfe59f1d1cf94f5406a/recovered.edits/0000000000000000004 2024-11-15T21:57:51,762 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1114): Opened aec2e46cb64898ba2a66825d3732f6dc; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62209343, jitterRate=-0.07300855219364166}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-15T21:57:51,762 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1122): Running coprocessor post-open hooks for aec2e46cb64898ba2a66825d3732f6dc 2024-11-15T21:57:51,762 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1006): Region open journal for aec2e46cb64898ba2a66825d3732f6dc: Running coprocessor pre-open hook at 1731707871476Writing region info on filesystem at 1731707871476Initializing all the Stores at 1731707871479 (+3 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707871479Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707871481 (+2 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707871481Obtaining lock to block concurrent updates at 1731707871519 (+38 ms)Preparing flush snapshotting stores in aec2e46cb64898ba2a66825d3732f6dc at 1731707871519Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731707682333.aec2e46cb64898ba2a66825d3732f6dc., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731707871519Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731707682333.aec2e46cb64898ba2a66825d3732f6dc. at 1731707871519Flushing aec2e46cb64898ba2a66825d3732f6dc/colfamily11: creating writer at 1731707871519Flushing aec2e46cb64898ba2a66825d3732f6dc/colfamily11: appending metadata at 1731707871617 (+98 ms)Flushing aec2e46cb64898ba2a66825d3732f6dc/colfamily11: closing flushed file at 1731707871617Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31053f01: reopening flushed file at 1731707871722 (+105 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for aec2e46cb64898ba2a66825d3732f6dc in 221ms, sequenceid=4, compaction requested=false; wal=null at 1731707871741 (+19 ms)Cleaning up temporary data from old regions at 1731707871743 (+2 ms)Running coprocessor post-open hooks at 1731707871762 (+19 ms)Region opened successfully at 1731707871762 2024-11-15T21:57:51,763 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731707682333.aec2e46cb64898ba2a66825d3732f6dc., pid=81, masterSystemTime=1731707871454 2024-11-15T21:57:51,765 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1048): stopping wal replay for 6e3b749fcbfdadfe59f1d1cf94f5406a 2024-11-15T21:57:51,765 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1060): Cleaning up temporary data for 6e3b749fcbfdadfe59f1d1cf94f5406a 2024-11-15T21:57:51,766 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-15T21:57:51,768 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1093): writing seq id for 6e3b749fcbfdadfe59f1d1cf94f5406a 2024-11-15T21:57:51,772 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731707682333.aec2e46cb64898ba2a66825d3732f6dc. 2024-11-15T21:57:51,772 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731707682333.aec2e46cb64898ba2a66825d3732f6dc. 2024-11-15T21:57:51,772 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731707682333.d6063c53955b74f03cc36bd578a6bd41. 2024-11-15T21:57:51,772 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6e3b749fcbfdadfe59f1d1cf94f5406a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T21:57:51,772 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7752): Opening region: {ENCODED => d6063c53955b74f03cc36bd578a6bd41, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731707682333.d6063c53955b74f03cc36bd578a6bd41.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-15T21:57:51,773 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true d6063c53955b74f03cc36bd578a6bd41 2024-11-15T21:57:51,773 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731707682333.d6063c53955b74f03cc36bd578a6bd41.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:57:51,773 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7794): checking encryption for d6063c53955b74f03cc36bd578a6bd41 2024-11-15T21:57:51,773 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7797): checking classloading for d6063c53955b74f03cc36bd578a6bd41 2024-11-15T21:57:51,778 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1114): Opened 6e3b749fcbfdadfe59f1d1cf94f5406a; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62713096, jitterRate=-0.06550204753875732}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-15T21:57:51,778 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6e3b749fcbfdadfe59f1d1cf94f5406a 2024-11-15T21:57:51,778 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=65 updating hbase:meta row=aec2e46cb64898ba2a66825d3732f6dc, regionState=OPEN, openSeqNum=5, regionLocation=ae3311ea5ccb,40941,1731707679516 2024-11-15T21:57:51,778 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1006): Region open journal for 6e3b749fcbfdadfe59f1d1cf94f5406a: Running coprocessor pre-open hook at 1731707871481Writing region info on filesystem at 1731707871481Initializing all the Stores at 1731707871482 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707871482Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707871482Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707871482Obtaining lock to block concurrent updates at 1731707871548 (+66 ms)Preparing flush snapshotting stores in 6e3b749fcbfdadfe59f1d1cf94f5406a at 1731707871548Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731707682333.6e3b749fcbfdadfe59f1d1cf94f5406a., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731707871548Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731707682333.6e3b749fcbfdadfe59f1d1cf94f5406a. at 1731707871548Flushing 6e3b749fcbfdadfe59f1d1cf94f5406a/colfamily11: creating writer at 1731707871548Flushing 6e3b749fcbfdadfe59f1d1cf94f5406a/colfamily11: appending metadata at 1731707871623 (+75 ms)Flushing 6e3b749fcbfdadfe59f1d1cf94f5406a/colfamily11: closing flushed file at 1731707871623Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a6b3d39: reopening flushed file at 1731707871741 (+118 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 6e3b749fcbfdadfe59f1d1cf94f5406a in 210ms, sequenceid=4, compaction requested=false; wal=null at 1731707871758 (+17 ms)Cleaning up temporary data from old regions at 1731707871765 (+7 ms)Running coprocessor post-open hooks at 1731707871778 (+13 ms)Region opened successfully at 1731707871778 2024-11-15T21:57:51,779 INFO [StoreOpener-d6063c53955b74f03cc36bd578a6bd41-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region d6063c53955b74f03cc36bd578a6bd41 2024-11-15T21:57:51,780 INFO [StoreOpener-d6063c53955b74f03cc36bd578a6bd41-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d6063c53955b74f03cc36bd578a6bd41 columnFamilyName colfamily11 2024-11-15T21:57:51,780 DEBUG [StoreOpener-d6063c53955b74f03cc36bd578a6bd41-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:57:51,781 INFO [StoreOpener-d6063c53955b74f03cc36bd578a6bd41-1 {}] regionserver.HStore(327): Store=d6063c53955b74f03cc36bd578a6bd41/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:57:51,781 INFO [StoreOpener-d6063c53955b74f03cc36bd578a6bd41-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region d6063c53955b74f03cc36bd578a6bd41 2024-11-15T21:57:51,782 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731707682333.6e3b749fcbfdadfe59f1d1cf94f5406a., pid=74, masterSystemTime=1731707871469 2024-11-15T21:57:51,782 INFO [StoreOpener-d6063c53955b74f03cc36bd578a6bd41-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d6063c53955b74f03cc36bd578a6bd41 columnFamilyName colfamily21 2024-11-15T21:57:51,782 DEBUG [StoreOpener-d6063c53955b74f03cc36bd578a6bd41-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:57:51,782 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=81, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure aec2e46cb64898ba2a66825d3732f6dc, server=ae3311ea5ccb,40941,1731707679516 because future has completed 2024-11-15T21:57:51,786 INFO [StoreOpener-d6063c53955b74f03cc36bd578a6bd41-1 {}] regionserver.HStore(327): Store=d6063c53955b74f03cc36bd578a6bd41/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:57:51,786 INFO [StoreOpener-d6063c53955b74f03cc36bd578a6bd41-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region d6063c53955b74f03cc36bd578a6bd41 2024-11-15T21:57:51,787 INFO [StoreOpener-d6063c53955b74f03cc36bd578a6bd41-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d6063c53955b74f03cc36bd578a6bd41 columnFamilyName colfamily31 2024-11-15T21:57:51,787 DEBUG [StoreOpener-d6063c53955b74f03cc36bd578a6bd41-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:57:51,787 INFO [StoreOpener-d6063c53955b74f03cc36bd578a6bd41-1 {}] regionserver.HStore(327): Store=d6063c53955b74f03cc36bd578a6bd41/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:57:51,788 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1038): replaying wal for d6063c53955b74f03cc36bd578a6bd41 2024-11-15T21:57:51,788 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d6063c53955b74f03cc36bd578a6bd41 2024-11-15T21:57:51,790 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731707682333.6e3b749fcbfdadfe59f1d1cf94f5406a. 2024-11-15T21:57:51,790 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731707682333.6e3b749fcbfdadfe59f1d1cf94f5406a. 2024-11-15T21:57:51,790 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d6063c53955b74f03cc36bd578a6bd41 2024-11-15T21:57:51,790 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731707682333.f9ef867a298d175eca9d85304f2faa69. 2024-11-15T21:57:51,791 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => f9ef867a298d175eca9d85304f2faa69, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731707682333.f9ef867a298d175eca9d85304f2faa69.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-15T21:57:51,791 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true f9ef867a298d175eca9d85304f2faa69 2024-11-15T21:57:51,791 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731707682333.f9ef867a298d175eca9d85304f2faa69.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:57:51,791 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for f9ef867a298d175eca9d85304f2faa69 2024-11-15T21:57:51,791 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for f9ef867a298d175eca9d85304f2faa69 2024-11-15T21:57:51,791 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d6063c53955b74f03cc36bd578a6bd41/recovered.edits/0000000000000000004 2024-11-15T21:57:51,793 INFO [StoreOpener-f9ef867a298d175eca9d85304f2faa69-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region f9ef867a298d175eca9d85304f2faa69 2024-11-15T21:57:51,796 INFO [StoreOpener-f9ef867a298d175eca9d85304f2faa69-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f9ef867a298d175eca9d85304f2faa69 columnFamilyName colfamily11 2024-11-15T21:57:51,796 DEBUG [StoreOpener-f9ef867a298d175eca9d85304f2faa69-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:57:51,797 INFO [StoreOpener-f9ef867a298d175eca9d85304f2faa69-1 {}] regionserver.HStore(327): Store=f9ef867a298d175eca9d85304f2faa69/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:57:51,797 INFO [StoreOpener-f9ef867a298d175eca9d85304f2faa69-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region f9ef867a298d175eca9d85304f2faa69 2024-11-15T21:57:51,798 INFO [StoreOpener-f9ef867a298d175eca9d85304f2faa69-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f9ef867a298d175eca9d85304f2faa69 columnFamilyName colfamily21 2024-11-15T21:57:51,798 DEBUG [StoreOpener-f9ef867a298d175eca9d85304f2faa69-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:57:51,799 INFO [StoreOpener-f9ef867a298d175eca9d85304f2faa69-1 {}] regionserver.HStore(327): Store=f9ef867a298d175eca9d85304f2faa69/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:57:51,799 INFO [StoreOpener-f9ef867a298d175eca9d85304f2faa69-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region f9ef867a298d175eca9d85304f2faa69 2024-11-15T21:57:51,802 INFO [StoreOpener-f9ef867a298d175eca9d85304f2faa69-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f9ef867a298d175eca9d85304f2faa69 columnFamilyName colfamily31 2024-11-15T21:57:51,802 DEBUG [StoreOpener-f9ef867a298d175eca9d85304f2faa69-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:57:51,802 INFO [StoreOpener-f9ef867a298d175eca9d85304f2faa69-1 {}] regionserver.HStore(327): Store=f9ef867a298d175eca9d85304f2faa69/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:57:51,803 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for f9ef867a298d175eca9d85304f2faa69 2024-11-15T21:57:51,803 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f9ef867a298d175eca9d85304f2faa69 2024-11-15T21:57:51,805 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f9ef867a298d175eca9d85304f2faa69 2024-11-15T21:57:51,806 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=65 2024-11-15T21:57:51,806 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=65, state=SUCCESS, hasLock=false; OpenRegionProcedure aec2e46cb64898ba2a66825d3732f6dc, server=ae3311ea5ccb,40941,1731707679516 in 448 msec 2024-11-15T21:57:51,807 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d6063c53955b74f03cc36bd578a6bd41/recovered.edits/0000000000000000004 2024-11-15T21:57:51,808 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(2902): Flushing d6063c53955b74f03cc36bd578a6bd41 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-15T21:57:51,808 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f9ef867a298d175eca9d85304f2faa69/recovered.edits/0000000000000000004 2024-11-15T21:57:51,809 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=aec2e46cb64898ba2a66825d3732f6dc, ASSIGN in 674 msec 2024-11-15T21:57:51,811 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=6e3b749fcbfdadfe59f1d1cf94f5406a, regionState=OPEN, openSeqNum=5, regionLocation=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:57:51,814 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=74, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6e3b749fcbfdadfe59f1d1cf94f5406a, server=ae3311ea5ccb,37455,1731707679725 because future has completed 2024-11-15T21:57:51,817 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f9ef867a298d175eca9d85304f2faa69/recovered.edits/0000000000000000004 2024-11-15T21:57:51,817 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(2902): Flushing f9ef867a298d175eca9d85304f2faa69 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-15T21:57:51,822 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=74, resume processing ppid=70 2024-11-15T21:57:51,822 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, ppid=70, state=SUCCESS, hasLock=false; OpenRegionProcedure 6e3b749fcbfdadfe59f1d1cf94f5406a, server=ae3311ea5ccb,37455,1731707679725 in 509 msec 2024-11-15T21:57:51,824 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=6e3b749fcbfdadfe59f1d1cf94f5406a, ASSIGN in 690 msec 2024-11-15T21:57:51,839 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d6063c53955b74f03cc36bd578a6bd41/.tmp/colfamily11/1932c4557a94484b9efeff587dc76402 is 52, key is ttt/colfamily11:colfamily11/1731707686843/Put/seqid=0 2024-11-15T21:57:51,848 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f9ef867a298d175eca9d85304f2faa69/.tmp/colfamily11/f7dfa49e3642491ead20c12d70d50cb3 is 52, key is ggg/colfamily11:colfamily11/1731707686431/Put/seqid=0 2024-11-15T21:57:51,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741880_1056 (size=39263) 2024-11-15T21:57:51,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741880_1056 (size=39263) 2024-11-15T21:57:51,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741881_1057 (size=39263) 2024-11-15T21:57:51,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741880_1056 (size=39263) 2024-11-15T21:57:51,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741881_1057 (size=39263) 2024-11-15T21:57:51,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741881_1057 (size=39263) 2024-11-15T21:57:51,973 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d6063c53955b74f03cc36bd578a6bd41/.tmp/colfamily11/1932c4557a94484b9efeff587dc76402 2024-11-15T21:57:51,982 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f9ef867a298d175eca9d85304f2faa69/.tmp/colfamily11/f7dfa49e3642491ead20c12d70d50cb3 2024-11-15T21:57:52,045 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d6063c53955b74f03cc36bd578a6bd41/.tmp/colfamily11/1932c4557a94484b9efeff587dc76402 as hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d6063c53955b74f03cc36bd578a6bd41/colfamily11/1932c4557a94484b9efeff587dc76402 2024-11-15T21:57:52,051 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f9ef867a298d175eca9d85304f2faa69/.tmp/colfamily11/f7dfa49e3642491ead20c12d70d50cb3 as hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f9ef867a298d175eca9d85304f2faa69/colfamily11/f7dfa49e3642491ead20c12d70d50cb3 2024-11-15T21:57:52,104 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d6063c53955b74f03cc36bd578a6bd41/colfamily11/1932c4557a94484b9efeff587dc76402, entries=703, sequenceid=4, filesize=38.3 K 2024-11-15T21:57:52,105 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for d6063c53955b74f03cc36bd578a6bd41 in 297ms, sequenceid=4, compaction requested=false; wal=null 2024-11-15T21:57:52,106 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f9ef867a298d175eca9d85304f2faa69/colfamily11/f7dfa49e3642491ead20c12d70d50cb3, entries=703, sequenceid=4, filesize=38.3 K 2024-11-15T21:57:52,106 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for f9ef867a298d175eca9d85304f2faa69 in 289ms, sequenceid=4, compaction requested=false; wal=null 2024-11-15T21:57:52,108 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d6063c53955b74f03cc36bd578a6bd41/recovered.edits/0000000000000000004 2024-11-15T21:57:52,111 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f9ef867a298d175eca9d85304f2faa69/recovered.edits/0000000000000000004 2024-11-15T21:57:52,111 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1048): stopping wal replay for d6063c53955b74f03cc36bd578a6bd41 2024-11-15T21:57:52,112 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1060): Cleaning up temporary data for d6063c53955b74f03cc36bd578a6bd41 2024-11-15T21:57:52,112 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-15T21:57:52,113 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for f9ef867a298d175eca9d85304f2faa69 2024-11-15T21:57:52,113 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for f9ef867a298d175eca9d85304f2faa69 2024-11-15T21:57:52,114 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-15T21:57:52,114 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1093): writing seq id for d6063c53955b74f03cc36bd578a6bd41 2024-11-15T21:57:52,116 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for f9ef867a298d175eca9d85304f2faa69 2024-11-15T21:57:52,118 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d6063c53955b74f03cc36bd578a6bd41/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T21:57:52,118 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f9ef867a298d175eca9d85304f2faa69/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T21:57:52,119 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1114): Opened d6063c53955b74f03cc36bd578a6bd41; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73820419, jitterRate=0.1000099629163742}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-15T21:57:52,119 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d6063c53955b74f03cc36bd578a6bd41 2024-11-15T21:57:52,120 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened f9ef867a298d175eca9d85304f2faa69; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68376061, jitterRate=0.01888270676136017}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-15T21:57:52,120 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f9ef867a298d175eca9d85304f2faa69 2024-11-15T21:57:52,120 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1006): Region open journal for d6063c53955b74f03cc36bd578a6bd41: Running coprocessor pre-open hook at 1731707871773Writing region info on filesystem at 1731707871773Initializing all the Stores at 1731707871778 (+5 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707871778Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707871778Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707871778Obtaining lock to block concurrent updates at 1731707871808 (+30 ms)Preparing flush snapshotting stores in d6063c53955b74f03cc36bd578a6bd41 at 1731707871808Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731707682333.d6063c53955b74f03cc36bd578a6bd41., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731707871808Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731707682333.d6063c53955b74f03cc36bd578a6bd41. at 1731707871808Flushing d6063c53955b74f03cc36bd578a6bd41/colfamily11: creating writer at 1731707871808Flushing d6063c53955b74f03cc36bd578a6bd41/colfamily11: appending metadata at 1731707871838 (+30 ms)Flushing d6063c53955b74f03cc36bd578a6bd41/colfamily11: closing flushed file at 1731707871838Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b35d4c1: reopening flushed file at 1731707872038 (+200 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for d6063c53955b74f03cc36bd578a6bd41 in 297ms, sequenceid=4, compaction requested=false; wal=null at 1731707872105 (+67 ms)Cleaning up temporary data from old regions at 1731707872112 (+7 ms)Running coprocessor post-open hooks at 1731707872120 (+8 ms)Region opened successfully at 1731707872120 2024-11-15T21:57:52,120 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for f9ef867a298d175eca9d85304f2faa69: Running coprocessor pre-open hook at 1731707871791Writing region info on filesystem at 1731707871791Initializing all the Stores at 1731707871793 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707871793Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707871793Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707871793Obtaining lock to block concurrent updates at 1731707871817 (+24 ms)Preparing flush snapshotting stores in f9ef867a298d175eca9d85304f2faa69 at 1731707871817Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731707682333.f9ef867a298d175eca9d85304f2faa69., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731707871817Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731707682333.f9ef867a298d175eca9d85304f2faa69. at 1731707871817Flushing f9ef867a298d175eca9d85304f2faa69/colfamily11: creating writer at 1731707871818 (+1 ms)Flushing f9ef867a298d175eca9d85304f2faa69/colfamily11: appending metadata at 1731707871848 (+30 ms)Flushing f9ef867a298d175eca9d85304f2faa69/colfamily11: closing flushed file at 1731707871848Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66f7aea0: reopening flushed file at 1731707872041 (+193 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for f9ef867a298d175eca9d85304f2faa69 in 289ms, sequenceid=4, compaction requested=false; wal=null at 1731707872106 (+65 ms)Cleaning up temporary data from old regions at 1731707872113 (+7 ms)Running coprocessor post-open hooks at 1731707872120 (+7 ms)Region opened successfully at 1731707872120 2024-11-15T21:57:52,121 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731707682333.f9ef867a298d175eca9d85304f2faa69., pid=76, masterSystemTime=1731707871469 2024-11-15T21:57:52,121 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731707682333.d6063c53955b74f03cc36bd578a6bd41., pid=80, masterSystemTime=1731707871454 2024-11-15T21:57:52,123 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731707682333.f9ef867a298d175eca9d85304f2faa69. 2024-11-15T21:57:52,123 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731707682333.f9ef867a298d175eca9d85304f2faa69. 2024-11-15T21:57:52,123 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731707682333.dd723478fcfc0e25b606965c9135244b. 2024-11-15T21:57:52,123 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => dd723478fcfc0e25b606965c9135244b, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731707682333.dd723478fcfc0e25b606965c9135244b.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-15T21:57:52,123 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true dd723478fcfc0e25b606965c9135244b 2024-11-15T21:57:52,124 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731707682333.dd723478fcfc0e25b606965c9135244b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:57:52,124 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for dd723478fcfc0e25b606965c9135244b 2024-11-15T21:57:52,124 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for dd723478fcfc0e25b606965c9135244b 2024-11-15T21:57:52,125 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=f9ef867a298d175eca9d85304f2faa69, regionState=OPEN, openSeqNum=5, regionLocation=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:57:52,125 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731707682333.d6063c53955b74f03cc36bd578a6bd41. 2024-11-15T21:57:52,125 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731707682333.d6063c53955b74f03cc36bd578a6bd41. 2024-11-15T21:57:52,125 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731707682333.3cf264afea186a5cc96edd3e68a10d45. 2024-11-15T21:57:52,125 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7752): Opening region: {ENCODED => 3cf264afea186a5cc96edd3e68a10d45, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731707682333.3cf264afea186a5cc96edd3e68a10d45.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-15T21:57:52,125 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 3cf264afea186a5cc96edd3e68a10d45 2024-11-15T21:57:52,126 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731707682333.3cf264afea186a5cc96edd3e68a10d45.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:57:52,126 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7794): checking encryption for 3cf264afea186a5cc96edd3e68a10d45 2024-11-15T21:57:52,126 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7797): checking classloading for 3cf264afea186a5cc96edd3e68a10d45 2024-11-15T21:57:52,127 INFO [StoreOpener-dd723478fcfc0e25b606965c9135244b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region dd723478fcfc0e25b606965c9135244b 2024-11-15T21:57:52,128 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=76, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure f9ef867a298d175eca9d85304f2faa69, server=ae3311ea5ccb,37455,1731707679725 because future has completed 2024-11-15T21:57:52,128 INFO [StoreOpener-dd723478fcfc0e25b606965c9135244b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dd723478fcfc0e25b606965c9135244b columnFamilyName colfamily11 2024-11-15T21:57:52,129 DEBUG [StoreOpener-dd723478fcfc0e25b606965c9135244b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:57:52,129 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=d6063c53955b74f03cc36bd578a6bd41, regionState=OPEN, openSeqNum=5, regionLocation=ae3311ea5ccb,40941,1731707679516 2024-11-15T21:57:52,129 INFO [StoreOpener-dd723478fcfc0e25b606965c9135244b-1 {}] regionserver.HStore(327): Store=dd723478fcfc0e25b606965c9135244b/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:57:52,129 INFO [StoreOpener-dd723478fcfc0e25b606965c9135244b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region dd723478fcfc0e25b606965c9135244b 2024-11-15T21:57:52,133 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36531 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=ae3311ea5ccb,37455,1731707679725, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=f9ef867a298d175eca9d85304f2faa69. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T21:57:52,135 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36531 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=ae3311ea5ccb,40941,1731707679516, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d6063c53955b74f03cc36bd578a6bd41. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T21:57:52,136 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=80, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure d6063c53955b74f03cc36bd578a6bd41, server=ae3311ea5ccb,40941,1731707679516 because future has completed 2024-11-15T21:57:52,137 INFO [StoreOpener-dd723478fcfc0e25b606965c9135244b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dd723478fcfc0e25b606965c9135244b columnFamilyName colfamily21 2024-11-15T21:57:52,137 DEBUG [StoreOpener-dd723478fcfc0e25b606965c9135244b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:57:52,138 INFO [StoreOpener-dd723478fcfc0e25b606965c9135244b-1 {}] regionserver.HStore(327): Store=dd723478fcfc0e25b606965c9135244b/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:57:52,138 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=76, resume processing ppid=66 2024-11-15T21:57:52,138 INFO [StoreOpener-dd723478fcfc0e25b606965c9135244b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region dd723478fcfc0e25b606965c9135244b 2024-11-15T21:57:52,138 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, ppid=66, state=SUCCESS, hasLock=false; OpenRegionProcedure f9ef867a298d175eca9d85304f2faa69, server=ae3311ea5ccb,37455,1731707679725 in 811 msec 2024-11-15T21:57:52,141 INFO [StoreOpener-dd723478fcfc0e25b606965c9135244b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dd723478fcfc0e25b606965c9135244b columnFamilyName colfamily31 2024-11-15T21:57:52,141 DEBUG [StoreOpener-dd723478fcfc0e25b606965c9135244b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:57:52,141 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=f9ef867a298d175eca9d85304f2faa69, ASSIGN in 1.0070 sec 2024-11-15T21:57:52,141 INFO [StoreOpener-dd723478fcfc0e25b606965c9135244b-1 {}] regionserver.HStore(327): Store=dd723478fcfc0e25b606965c9135244b/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:57:52,141 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for dd723478fcfc0e25b606965c9135244b 2024-11-15T21:57:52,143 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dd723478fcfc0e25b606965c9135244b 2024-11-15T21:57:52,146 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=71 2024-11-15T21:57:52,146 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=71, state=SUCCESS, hasLock=false; OpenRegionProcedure d6063c53955b74f03cc36bd578a6bd41, server=ae3311ea5ccb,40941,1731707679516 in 805 msec 2024-11-15T21:57:52,146 INFO [StoreOpener-3cf264afea186a5cc96edd3e68a10d45-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 3cf264afea186a5cc96edd3e68a10d45 2024-11-15T21:57:52,148 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d6063c53955b74f03cc36bd578a6bd41, ASSIGN in 1.0140 sec 2024-11-15T21:57:52,148 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dd723478fcfc0e25b606965c9135244b 2024-11-15T21:57:52,148 INFO [StoreOpener-3cf264afea186a5cc96edd3e68a10d45-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3cf264afea186a5cc96edd3e68a10d45 columnFamilyName colfamily11 2024-11-15T21:57:52,148 DEBUG [StoreOpener-3cf264afea186a5cc96edd3e68a10d45-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:57:52,149 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dd723478fcfc0e25b606965c9135244b/recovered.edits/0000000000000000004 2024-11-15T21:57:52,149 INFO [StoreOpener-3cf264afea186a5cc96edd3e68a10d45-1 {}] regionserver.HStore(327): Store=3cf264afea186a5cc96edd3e68a10d45/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:57:52,150 INFO [StoreOpener-3cf264afea186a5cc96edd3e68a10d45-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 3cf264afea186a5cc96edd3e68a10d45 2024-11-15T21:57:52,151 INFO [StoreOpener-3cf264afea186a5cc96edd3e68a10d45-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3cf264afea186a5cc96edd3e68a10d45 columnFamilyName colfamily21 2024-11-15T21:57:52,151 DEBUG [StoreOpener-3cf264afea186a5cc96edd3e68a10d45-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:57:52,152 INFO [StoreOpener-3cf264afea186a5cc96edd3e68a10d45-1 {}] regionserver.HStore(327): Store=3cf264afea186a5cc96edd3e68a10d45/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:57:52,152 INFO [StoreOpener-3cf264afea186a5cc96edd3e68a10d45-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 3cf264afea186a5cc96edd3e68a10d45 2024-11-15T21:57:52,153 INFO [StoreOpener-3cf264afea186a5cc96edd3e68a10d45-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3cf264afea186a5cc96edd3e68a10d45 columnFamilyName colfamily31 2024-11-15T21:57:52,153 DEBUG [StoreOpener-3cf264afea186a5cc96edd3e68a10d45-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:57:52,157 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dd723478fcfc0e25b606965c9135244b/recovered.edits/0000000000000000004 2024-11-15T21:57:52,157 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(2902): Flushing dd723478fcfc0e25b606965c9135244b 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-15T21:57:52,154 INFO [StoreOpener-3cf264afea186a5cc96edd3e68a10d45-1 {}] regionserver.HStore(327): Store=3cf264afea186a5cc96edd3e68a10d45/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:57:52,157 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1038): replaying wal for 3cf264afea186a5cc96edd3e68a10d45 2024-11-15T21:57:52,158 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3cf264afea186a5cc96edd3e68a10d45 2024-11-15T21:57:52,160 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3cf264afea186a5cc96edd3e68a10d45 2024-11-15T21:57:52,161 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3cf264afea186a5cc96edd3e68a10d45/recovered.edits/0000000000000000004 2024-11-15T21:57:52,169 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3cf264afea186a5cc96edd3e68a10d45/recovered.edits/0000000000000000004 2024-11-15T21:57:52,169 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(2902): Flushing 3cf264afea186a5cc96edd3e68a10d45 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-15T21:57:52,183 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dd723478fcfc0e25b606965c9135244b/.tmp/colfamily11/9d9f742e8452494087db1876c7389596 is 52, key is bbb/colfamily11:colfamily11/1731707686325/Put/seqid=0 2024-11-15T21:57:52,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741882_1058 (size=39263) 2024-11-15T21:57:52,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741882_1058 (size=39263) 2024-11-15T21:57:52,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741882_1058 (size=39263) 2024-11-15T21:57:52,194 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dd723478fcfc0e25b606965c9135244b/.tmp/colfamily11/9d9f742e8452494087db1876c7389596 2024-11-15T21:57:52,197 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3cf264afea186a5cc96edd3e68a10d45/.tmp/colfamily11/df2238e92f2d4accb834a5df11a21caf is 52, key is ppp/colfamily11:colfamily11/1731707686735/Put/seqid=0 2024-11-15T21:57:52,204 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dd723478fcfc0e25b606965c9135244b/.tmp/colfamily11/9d9f742e8452494087db1876c7389596 as hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dd723478fcfc0e25b606965c9135244b/colfamily11/9d9f742e8452494087db1876c7389596 2024-11-15T21:57:52,213 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dd723478fcfc0e25b606965c9135244b/colfamily11/9d9f742e8452494087db1876c7389596, entries=703, sequenceid=4, filesize=38.3 K 2024-11-15T21:57:52,213 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for dd723478fcfc0e25b606965c9135244b in 56ms, sequenceid=4, compaction requested=false; wal=null 2024-11-15T21:57:52,214 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dd723478fcfc0e25b606965c9135244b/recovered.edits/0000000000000000004 2024-11-15T21:57:52,215 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for dd723478fcfc0e25b606965c9135244b 2024-11-15T21:57:52,215 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for dd723478fcfc0e25b606965c9135244b 2024-11-15T21:57:52,216 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-15T21:57:52,218 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for dd723478fcfc0e25b606965c9135244b 2024-11-15T21:57:52,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741883_1059 (size=39263) 2024-11-15T21:57:52,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741883_1059 (size=39263) 2024-11-15T21:57:52,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741883_1059 (size=39263) 2024-11-15T21:57:52,223 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dd723478fcfc0e25b606965c9135244b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T21:57:52,223 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3cf264afea186a5cc96edd3e68a10d45/.tmp/colfamily11/df2238e92f2d4accb834a5df11a21caf 2024-11-15T21:57:52,226 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened dd723478fcfc0e25b606965c9135244b; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65572559, jitterRate=-0.022892728447914124}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-15T21:57:52,226 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dd723478fcfc0e25b606965c9135244b 2024-11-15T21:57:52,226 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for dd723478fcfc0e25b606965c9135244b: Running coprocessor pre-open hook at 1731707872124Writing region info on filesystem at 1731707872124Initializing all the Stores at 1731707872125 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707872125Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707872127 (+2 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707872127Obtaining lock to block concurrent updates at 1731707872157 (+30 ms)Preparing flush snapshotting stores in dd723478fcfc0e25b606965c9135244b at 1731707872157Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731707682333.dd723478fcfc0e25b606965c9135244b., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731707872157Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731707682333.dd723478fcfc0e25b606965c9135244b. at 1731707872157Flushing dd723478fcfc0e25b606965c9135244b/colfamily11: creating writer at 1731707872157Flushing dd723478fcfc0e25b606965c9135244b/colfamily11: appending metadata at 1731707872182 (+25 ms)Flushing dd723478fcfc0e25b606965c9135244b/colfamily11: closing flushed file at 1731707872182Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@58898154: reopening flushed file at 1731707872203 (+21 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for dd723478fcfc0e25b606965c9135244b in 56ms, sequenceid=4, compaction requested=false; wal=null at 1731707872213 (+10 ms)Cleaning up temporary data from old regions at 1731707872215 (+2 ms)Running coprocessor post-open hooks at 1731707872226 (+11 ms)Region opened successfully at 1731707872226 2024-11-15T21:57:52,228 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731707682333.dd723478fcfc0e25b606965c9135244b., pid=75, masterSystemTime=1731707871469 2024-11-15T21:57:52,231 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731707682333.dd723478fcfc0e25b606965c9135244b. 2024-11-15T21:57:52,231 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731707682333.dd723478fcfc0e25b606965c9135244b. 2024-11-15T21:57:52,231 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731707682333.d9989b0341ef96818e811f874dbcf91c. 2024-11-15T21:57:52,231 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => d9989b0341ef96818e811f874dbcf91c, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731707682333.d9989b0341ef96818e811f874dbcf91c.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-15T21:57:52,232 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true d9989b0341ef96818e811f874dbcf91c 2024-11-15T21:57:52,232 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731707682333.d9989b0341ef96818e811f874dbcf91c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:57:52,232 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for d9989b0341ef96818e811f874dbcf91c 2024-11-15T21:57:52,232 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for d9989b0341ef96818e811f874dbcf91c 2024-11-15T21:57:52,232 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=dd723478fcfc0e25b606965c9135244b, regionState=OPEN, openSeqNum=5, regionLocation=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:57:52,234 INFO [StoreOpener-d9989b0341ef96818e811f874dbcf91c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region d9989b0341ef96818e811f874dbcf91c 2024-11-15T21:57:52,235 INFO [StoreOpener-d9989b0341ef96818e811f874dbcf91c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9989b0341ef96818e811f874dbcf91c columnFamilyName colfamily11 2024-11-15T21:57:52,235 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3cf264afea186a5cc96edd3e68a10d45/.tmp/colfamily11/df2238e92f2d4accb834a5df11a21caf as hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3cf264afea186a5cc96edd3e68a10d45/colfamily11/df2238e92f2d4accb834a5df11a21caf 2024-11-15T21:57:52,236 DEBUG [StoreOpener-d9989b0341ef96818e811f874dbcf91c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:57:52,236 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36531 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=ae3311ea5ccb,37455,1731707679725, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=dd723478fcfc0e25b606965c9135244b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T21:57:52,237 INFO [StoreOpener-d9989b0341ef96818e811f874dbcf91c-1 {}] regionserver.HStore(327): Store=d9989b0341ef96818e811f874dbcf91c/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:57:52,237 INFO [StoreOpener-d9989b0341ef96818e811f874dbcf91c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region d9989b0341ef96818e811f874dbcf91c 2024-11-15T21:57:52,242 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=75, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure dd723478fcfc0e25b606965c9135244b, server=ae3311ea5ccb,37455,1731707679725 because future has completed 2024-11-15T21:57:52,244 INFO [StoreOpener-d9989b0341ef96818e811f874dbcf91c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9989b0341ef96818e811f874dbcf91c columnFamilyName colfamily21 2024-11-15T21:57:52,244 DEBUG [StoreOpener-d9989b0341ef96818e811f874dbcf91c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:57:52,246 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=64 2024-11-15T21:57:52,246 INFO [StoreOpener-d9989b0341ef96818e811f874dbcf91c-1 {}] regionserver.HStore(327): Store=d9989b0341ef96818e811f874dbcf91c/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:57:52,246 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure dd723478fcfc0e25b606965c9135244b, server=ae3311ea5ccb,37455,1731707679725 in 927 msec 2024-11-15T21:57:52,247 INFO [StoreOpener-d9989b0341ef96818e811f874dbcf91c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region d9989b0341ef96818e811f874dbcf91c 2024-11-15T21:57:52,248 INFO [StoreOpener-d9989b0341ef96818e811f874dbcf91c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9989b0341ef96818e811f874dbcf91c columnFamilyName colfamily31 2024-11-15T21:57:52,249 DEBUG [StoreOpener-d9989b0341ef96818e811f874dbcf91c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:57:52,249 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=dd723478fcfc0e25b606965c9135244b, ASSIGN in 1.1150 sec 2024-11-15T21:57:52,251 INFO [StoreOpener-d9989b0341ef96818e811f874dbcf91c-1 {}] regionserver.HStore(327): Store=d9989b0341ef96818e811f874dbcf91c/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:57:52,251 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for d9989b0341ef96818e811f874dbcf91c 2024-11-15T21:57:52,252 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d9989b0341ef96818e811f874dbcf91c 2024-11-15T21:57:52,253 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3cf264afea186a5cc96edd3e68a10d45/colfamily11/df2238e92f2d4accb834a5df11a21caf, entries=703, sequenceid=4, filesize=38.3 K 2024-11-15T21:57:52,253 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 3cf264afea186a5cc96edd3e68a10d45 in 84ms, sequenceid=4, compaction requested=false; wal=null 2024-11-15T21:57:52,254 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d9989b0341ef96818e811f874dbcf91c 2024-11-15T21:57:52,254 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3cf264afea186a5cc96edd3e68a10d45/recovered.edits/0000000000000000004 2024-11-15T21:57:52,255 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d9989b0341ef96818e811f874dbcf91c/recovered.edits/0000000000000000004 2024-11-15T21:57:52,256 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1048): stopping wal replay for 3cf264afea186a5cc96edd3e68a10d45 2024-11-15T21:57:52,256 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1060): Cleaning up temporary data for 3cf264afea186a5cc96edd3e68a10d45 2024-11-15T21:57:52,257 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-15T21:57:52,261 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d9989b0341ef96818e811f874dbcf91c/recovered.edits/0000000000000000004 2024-11-15T21:57:52,261 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(2902): Flushing d9989b0341ef96818e811f874dbcf91c 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-15T21:57:52,263 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1093): writing seq id for 3cf264afea186a5cc96edd3e68a10d45 2024-11-15T21:57:52,273 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3cf264afea186a5cc96edd3e68a10d45/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T21:57:52,275 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1114): Opened 3cf264afea186a5cc96edd3e68a10d45; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65977748, jitterRate=-0.016854941844940186}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-15T21:57:52,275 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3cf264afea186a5cc96edd3e68a10d45 2024-11-15T21:57:52,275 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1006): Region open journal for 3cf264afea186a5cc96edd3e68a10d45: Running coprocessor pre-open hook at 1731707872126Writing region info on filesystem at 1731707872126Initializing all the Stores at 1731707872128 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707872128Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707872146 (+18 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707872146Obtaining lock to block concurrent updates at 1731707872169 (+23 ms)Preparing flush snapshotting stores in 3cf264afea186a5cc96edd3e68a10d45 at 1731707872169Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731707682333.3cf264afea186a5cc96edd3e68a10d45., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731707872170 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731707682333.3cf264afea186a5cc96edd3e68a10d45. at 1731707872170Flushing 3cf264afea186a5cc96edd3e68a10d45/colfamily11: creating writer at 1731707872170Flushing 3cf264afea186a5cc96edd3e68a10d45/colfamily11: appending metadata at 1731707872196 (+26 ms)Flushing 3cf264afea186a5cc96edd3e68a10d45/colfamily11: closing flushed file at 1731707872196Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@30c1ccff: reopening flushed file at 1731707872234 (+38 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 3cf264afea186a5cc96edd3e68a10d45 in 84ms, sequenceid=4, compaction requested=false; wal=null at 1731707872253 (+19 ms)Cleaning up temporary data from old regions at 1731707872256 (+3 ms)Running coprocessor post-open hooks at 1731707872275 (+19 ms)Region opened successfully at 1731707872275 2024-11-15T21:57:52,277 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731707682333.3cf264afea186a5cc96edd3e68a10d45., pid=73, masterSystemTime=1731707871454 2024-11-15T21:57:52,280 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731707682333.3cf264afea186a5cc96edd3e68a10d45. 2024-11-15T21:57:52,280 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731707682333.3cf264afea186a5cc96edd3e68a10d45. 2024-11-15T21:57:52,280 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=3cf264afea186a5cc96edd3e68a10d45, regionState=OPEN, openSeqNum=5, regionLocation=ae3311ea5ccb,40941,1731707679516 2024-11-15T21:57:52,280 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731707682333.4256fff2fa33299a6264dd460a4003fb. 2024-11-15T21:57:52,280 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => 4256fff2fa33299a6264dd460a4003fb, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731707682333.4256fff2fa33299a6264dd460a4003fb.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-15T21:57:52,281 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 4256fff2fa33299a6264dd460a4003fb 2024-11-15T21:57:52,281 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731707682333.4256fff2fa33299a6264dd460a4003fb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:57:52,281 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for 4256fff2fa33299a6264dd460a4003fb 2024-11-15T21:57:52,281 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for 4256fff2fa33299a6264dd460a4003fb 2024-11-15T21:57:52,283 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=73, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3cf264afea186a5cc96edd3e68a10d45, server=ae3311ea5ccb,40941,1731707679516 because future has completed 2024-11-15T21:57:52,288 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d9989b0341ef96818e811f874dbcf91c/.tmp/colfamily11/1f8229f0250f4dab9120d3c8f904d1b0 is 52, key is rrr/colfamily11:colfamily11/1731707686785/Put/seqid=0 2024-11-15T21:57:52,289 INFO [StoreOpener-4256fff2fa33299a6264dd460a4003fb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 4256fff2fa33299a6264dd460a4003fb 2024-11-15T21:57:52,297 INFO [StoreOpener-4256fff2fa33299a6264dd460a4003fb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4256fff2fa33299a6264dd460a4003fb columnFamilyName colfamily11 2024-11-15T21:57:52,297 DEBUG [StoreOpener-4256fff2fa33299a6264dd460a4003fb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:57:52,298 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=67 2024-11-15T21:57:52,299 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=67, state=SUCCESS, hasLock=false; OpenRegionProcedure 3cf264afea186a5cc96edd3e68a10d45, server=ae3311ea5ccb,40941,1731707679516 in 991 msec 2024-11-15T21:57:52,301 INFO [StoreOpener-4256fff2fa33299a6264dd460a4003fb-1 {}] regionserver.HStore(327): Store=4256fff2fa33299a6264dd460a4003fb/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:57:52,301 INFO [StoreOpener-4256fff2fa33299a6264dd460a4003fb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 4256fff2fa33299a6264dd460a4003fb 2024-11-15T21:57:52,301 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3cf264afea186a5cc96edd3e68a10d45, ASSIGN in 1.1670 sec 2024-11-15T21:57:52,302 INFO [StoreOpener-4256fff2fa33299a6264dd460a4003fb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4256fff2fa33299a6264dd460a4003fb columnFamilyName colfamily21 2024-11-15T21:57:52,302 DEBUG [StoreOpener-4256fff2fa33299a6264dd460a4003fb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:57:52,302 INFO [StoreOpener-4256fff2fa33299a6264dd460a4003fb-1 {}] regionserver.HStore(327): Store=4256fff2fa33299a6264dd460a4003fb/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:57:52,303 INFO [StoreOpener-4256fff2fa33299a6264dd460a4003fb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 4256fff2fa33299a6264dd460a4003fb 2024-11-15T21:57:52,303 INFO [StoreOpener-4256fff2fa33299a6264dd460a4003fb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4256fff2fa33299a6264dd460a4003fb columnFamilyName colfamily31 2024-11-15T21:57:52,303 DEBUG [StoreOpener-4256fff2fa33299a6264dd460a4003fb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:57:52,304 INFO [StoreOpener-4256fff2fa33299a6264dd460a4003fb-1 {}] regionserver.HStore(327): Store=4256fff2fa33299a6264dd460a4003fb/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:57:52,305 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for 4256fff2fa33299a6264dd460a4003fb 2024-11-15T21:57:52,306 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4256fff2fa33299a6264dd460a4003fb 2024-11-15T21:57:52,308 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4256fff2fa33299a6264dd460a4003fb 2024-11-15T21:57:52,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741884_1060 (size=39263) 2024-11-15T21:57:52,309 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4256fff2fa33299a6264dd460a4003fb/recovered.edits/0000000000000000004 2024-11-15T21:57:52,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741884_1060 (size=39263) 2024-11-15T21:57:52,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741884_1060 (size=39263) 2024-11-15T21:57:52,318 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d9989b0341ef96818e811f874dbcf91c/.tmp/colfamily11/1f8229f0250f4dab9120d3c8f904d1b0 2024-11-15T21:57:52,329 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4256fff2fa33299a6264dd460a4003fb/recovered.edits/0000000000000000004 2024-11-15T21:57:52,329 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(2902): Flushing 4256fff2fa33299a6264dd460a4003fb 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-15T21:57:52,340 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d9989b0341ef96818e811f874dbcf91c/.tmp/colfamily11/1f8229f0250f4dab9120d3c8f904d1b0 as hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d9989b0341ef96818e811f874dbcf91c/colfamily11/1f8229f0250f4dab9120d3c8f904d1b0 2024-11-15T21:57:52,351 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d9989b0341ef96818e811f874dbcf91c/colfamily11/1f8229f0250f4dab9120d3c8f904d1b0, entries=703, sequenceid=4, filesize=38.3 K 2024-11-15T21:57:52,352 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for d9989b0341ef96818e811f874dbcf91c in 91ms, sequenceid=4, compaction requested=false; wal=null 2024-11-15T21:57:52,360 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d9989b0341ef96818e811f874dbcf91c/recovered.edits/0000000000000000004 2024-11-15T21:57:52,360 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4256fff2fa33299a6264dd460a4003fb/.tmp/colfamily11/ab3c11bbdd8f4deda50f97d277832ae7 is 52, key is ddd/colfamily11:colfamily11/1731707686381/Put/seqid=0 2024-11-15T21:57:52,363 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for d9989b0341ef96818e811f874dbcf91c 2024-11-15T21:57:52,363 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for d9989b0341ef96818e811f874dbcf91c 2024-11-15T21:57:52,368 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-15T21:57:52,373 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for d9989b0341ef96818e811f874dbcf91c 2024-11-15T21:57:52,410 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d9989b0341ef96818e811f874dbcf91c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T21:57:52,411 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened d9989b0341ef96818e811f874dbcf91c; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72032589, jitterRate=0.07336921989917755}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-15T21:57:52,411 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d9989b0341ef96818e811f874dbcf91c 2024-11-15T21:57:52,412 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for d9989b0341ef96818e811f874dbcf91c: Running coprocessor pre-open hook at 1731707872232Writing region info on filesystem at 1731707872232Initializing all the Stores at 1731707872233 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707872233Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707872234 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707872234Obtaining lock to block concurrent updates at 1731707872261 (+27 ms)Preparing flush snapshotting stores in d9989b0341ef96818e811f874dbcf91c at 1731707872261Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731707682333.d9989b0341ef96818e811f874dbcf91c., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731707872261Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731707682333.d9989b0341ef96818e811f874dbcf91c. at 1731707872261Flushing d9989b0341ef96818e811f874dbcf91c/colfamily11: creating writer at 1731707872261Flushing d9989b0341ef96818e811f874dbcf91c/colfamily11: appending metadata at 1731707872287 (+26 ms)Flushing d9989b0341ef96818e811f874dbcf91c/colfamily11: closing flushed file at 1731707872287Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e126f1b: reopening flushed file at 1731707872339 (+52 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for d9989b0341ef96818e811f874dbcf91c in 91ms, sequenceid=4, compaction requested=false; wal=null at 1731707872352 (+13 ms)Cleaning up temporary data from old regions at 1731707872363 (+11 ms)Running coprocessor post-open hooks at 1731707872411 (+48 ms)Region opened successfully at 1731707872412 (+1 ms) 2024-11-15T21:57:52,414 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731707682333.d9989b0341ef96818e811f874dbcf91c., pid=77, masterSystemTime=1731707871469 2024-11-15T21:57:52,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741885_1061 (size=39263) 2024-11-15T21:57:52,423 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731707682333.d9989b0341ef96818e811f874dbcf91c. 2024-11-15T21:57:52,424 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731707682333.d9989b0341ef96818e811f874dbcf91c. 2024-11-15T21:57:52,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741885_1061 (size=39263) 2024-11-15T21:57:52,424 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731707682333.6694325f9c862a5a1ee02eabf7861cc3. 2024-11-15T21:57:52,424 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => 6694325f9c862a5a1ee02eabf7861cc3, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731707682333.6694325f9c862a5a1ee02eabf7861cc3.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-15T21:57:52,424 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 6694325f9c862a5a1ee02eabf7861cc3 2024-11-15T21:57:52,425 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731707682333.6694325f9c862a5a1ee02eabf7861cc3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:57:52,425 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for 6694325f9c862a5a1ee02eabf7861cc3 2024-11-15T21:57:52,425 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for 6694325f9c862a5a1ee02eabf7861cc3 2024-11-15T21:57:52,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741885_1061 (size=39263) 2024-11-15T21:57:52,430 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=d9989b0341ef96818e811f874dbcf91c, regionState=OPEN, openSeqNum=5, regionLocation=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:57:52,432 INFO [StoreOpener-6694325f9c862a5a1ee02eabf7861cc3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 6694325f9c862a5a1ee02eabf7861cc3 2024-11-15T21:57:52,435 INFO [StoreOpener-6694325f9c862a5a1ee02eabf7861cc3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6694325f9c862a5a1ee02eabf7861cc3 columnFamilyName colfamily11 2024-11-15T21:57:52,435 DEBUG [StoreOpener-6694325f9c862a5a1ee02eabf7861cc3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:57:52,436 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure d9989b0341ef96818e811f874dbcf91c, server=ae3311ea5ccb,37455,1731707679725 because future has completed 2024-11-15T21:57:52,440 INFO [StoreOpener-6694325f9c862a5a1ee02eabf7861cc3-1 {}] regionserver.HStore(327): Store=6694325f9c862a5a1ee02eabf7861cc3/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:57:52,441 INFO [StoreOpener-6694325f9c862a5a1ee02eabf7861cc3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 6694325f9c862a5a1ee02eabf7861cc3 2024-11-15T21:57:52,446 INFO [StoreOpener-6694325f9c862a5a1ee02eabf7861cc3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6694325f9c862a5a1ee02eabf7861cc3 columnFamilyName colfamily21 2024-11-15T21:57:52,446 DEBUG [StoreOpener-6694325f9c862a5a1ee02eabf7861cc3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:57:52,446 INFO [StoreOpener-6694325f9c862a5a1ee02eabf7861cc3-1 {}] regionserver.HStore(327): Store=6694325f9c862a5a1ee02eabf7861cc3/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:57:52,447 INFO [StoreOpener-6694325f9c862a5a1ee02eabf7861cc3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 6694325f9c862a5a1ee02eabf7861cc3 2024-11-15T21:57:52,448 INFO [StoreOpener-6694325f9c862a5a1ee02eabf7861cc3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6694325f9c862a5a1ee02eabf7861cc3 columnFamilyName colfamily31 2024-11-15T21:57:52,448 DEBUG [StoreOpener-6694325f9c862a5a1ee02eabf7861cc3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:57:52,449 INFO [StoreOpener-6694325f9c862a5a1ee02eabf7861cc3-1 {}] regionserver.HStore(327): Store=6694325f9c862a5a1ee02eabf7861cc3/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:57:52,449 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for 6694325f9c862a5a1ee02eabf7861cc3 2024-11-15T21:57:52,451 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6694325f9c862a5a1ee02eabf7861cc3 2024-11-15T21:57:52,452 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=69 2024-11-15T21:57:52,452 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=69, state=SUCCESS, hasLock=false; OpenRegionProcedure d9989b0341ef96818e811f874dbcf91c, server=ae3311ea5ccb,37455,1731707679725 in 1.1080 sec 2024-11-15T21:57:52,455 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6694325f9c862a5a1ee02eabf7861cc3 2024-11-15T21:57:52,459 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6694325f9c862a5a1ee02eabf7861cc3/recovered.edits/0000000000000000004 2024-11-15T21:57:52,465 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d9989b0341ef96818e811f874dbcf91c, ASSIGN in 1.3200 sec 2024-11-15T21:57:52,465 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6694325f9c862a5a1ee02eabf7861cc3/recovered.edits/0000000000000000004 2024-11-15T21:57:52,466 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(2902): Flushing 6694325f9c862a5a1ee02eabf7861cc3 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-15T21:57:52,500 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6694325f9c862a5a1ee02eabf7861cc3/.tmp/colfamily11/efdfa7eb43ae41d4959ff33ac51e8a13 is 52, key is www/colfamily11:colfamily11/1731707686919/Put/seqid=0 2024-11-15T21:57:52,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741886_1062 (size=39263) 2024-11-15T21:57:52,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741886_1062 (size=39263) 2024-11-15T21:57:52,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741886_1062 (size=39263) 2024-11-15T21:57:52,595 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6694325f9c862a5a1ee02eabf7861cc3/.tmp/colfamily11/efdfa7eb43ae41d4959ff33ac51e8a13 2024-11-15T21:57:52,612 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6694325f9c862a5a1ee02eabf7861cc3/.tmp/colfamily11/efdfa7eb43ae41d4959ff33ac51e8a13 as hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6694325f9c862a5a1ee02eabf7861cc3/colfamily11/efdfa7eb43ae41d4959ff33ac51e8a13 2024-11-15T21:57:52,627 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6694325f9c862a5a1ee02eabf7861cc3/colfamily11/efdfa7eb43ae41d4959ff33ac51e8a13, entries=703, sequenceid=4, filesize=38.3 K 2024-11-15T21:57:52,627 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 6694325f9c862a5a1ee02eabf7861cc3 in 162ms, sequenceid=4, compaction requested=false; wal=null 2024-11-15T21:57:52,629 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6694325f9c862a5a1ee02eabf7861cc3/recovered.edits/0000000000000000004 2024-11-15T21:57:52,630 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for 6694325f9c862a5a1ee02eabf7861cc3 2024-11-15T21:57:52,630 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for 6694325f9c862a5a1ee02eabf7861cc3 2024-11-15T21:57:52,631 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-15T21:57:52,633 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for 6694325f9c862a5a1ee02eabf7861cc3 2024-11-15T21:57:52,654 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6694325f9c862a5a1ee02eabf7861cc3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T21:57:52,657 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened 6694325f9c862a5a1ee02eabf7861cc3; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62339974, jitterRate=-0.07106199860572815}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-15T21:57:52,657 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6694325f9c862a5a1ee02eabf7861cc3 2024-11-15T21:57:52,658 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for 6694325f9c862a5a1ee02eabf7861cc3: Running coprocessor pre-open hook at 1731707872425Writing region info on filesystem at 1731707872425Initializing all the Stores at 1731707872431 (+6 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707872431Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707872432 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707872432Obtaining lock to block concurrent updates at 1731707872466 (+34 ms)Preparing flush snapshotting stores in 6694325f9c862a5a1ee02eabf7861cc3 at 1731707872466Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731707682333.6694325f9c862a5a1ee02eabf7861cc3., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731707872466Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731707682333.6694325f9c862a5a1ee02eabf7861cc3. at 1731707872466Flushing 6694325f9c862a5a1ee02eabf7861cc3/colfamily11: creating writer at 1731707872466Flushing 6694325f9c862a5a1ee02eabf7861cc3/colfamily11: appending metadata at 1731707872499 (+33 ms)Flushing 6694325f9c862a5a1ee02eabf7861cc3/colfamily11: closing flushed file at 1731707872499Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a46658a: reopening flushed file at 1731707872606 (+107 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 6694325f9c862a5a1ee02eabf7861cc3 in 162ms, sequenceid=4, compaction requested=false; wal=null at 1731707872627 (+21 ms)Cleaning up temporary data from old regions at 1731707872630 (+3 ms)Running coprocessor post-open hooks at 1731707872657 (+27 ms)Region opened successfully at 1731707872657 2024-11-15T21:57:52,661 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731707682333.6694325f9c862a5a1ee02eabf7861cc3., pid=78, masterSystemTime=1731707871469 2024-11-15T21:57:52,666 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731707682333.6694325f9c862a5a1ee02eabf7861cc3. 2024-11-15T21:57:52,666 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731707682333.6694325f9c862a5a1ee02eabf7861cc3. 2024-11-15T21:57:52,670 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=6694325f9c862a5a1ee02eabf7861cc3, regionState=OPEN, openSeqNum=5, regionLocation=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:57:52,673 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6694325f9c862a5a1ee02eabf7861cc3, server=ae3311ea5ccb,37455,1731707679725 because future has completed 2024-11-15T21:57:52,709 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=78, resume processing ppid=72 2024-11-15T21:57:52,710 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, ppid=72, state=SUCCESS, hasLock=false; OpenRegionProcedure 6694325f9c862a5a1ee02eabf7861cc3, server=ae3311ea5ccb,37455,1731707679725 in 1.3530 sec 2024-11-15T21:57:52,722 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=6694325f9c862a5a1ee02eabf7861cc3, ASSIGN in 1.5780 sec 2024-11-15T21:57:52,824 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4256fff2fa33299a6264dd460a4003fb/.tmp/colfamily11/ab3c11bbdd8f4deda50f97d277832ae7 2024-11-15T21:57:52,833 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4256fff2fa33299a6264dd460a4003fb/.tmp/colfamily11/ab3c11bbdd8f4deda50f97d277832ae7 as hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4256fff2fa33299a6264dd460a4003fb/colfamily11/ab3c11bbdd8f4deda50f97d277832ae7 2024-11-15T21:57:52,841 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4256fff2fa33299a6264dd460a4003fb/colfamily11/ab3c11bbdd8f4deda50f97d277832ae7, entries=703, sequenceid=4, filesize=38.3 K 2024-11-15T21:57:52,842 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 4256fff2fa33299a6264dd460a4003fb in 513ms, sequenceid=4, compaction requested=false; wal=null 2024-11-15T21:57:52,843 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4256fff2fa33299a6264dd460a4003fb/recovered.edits/0000000000000000004 2024-11-15T21:57:52,844 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for 4256fff2fa33299a6264dd460a4003fb 2024-11-15T21:57:52,844 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for 4256fff2fa33299a6264dd460a4003fb 2024-11-15T21:57:52,845 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-15T21:57:52,852 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for 4256fff2fa33299a6264dd460a4003fb 2024-11-15T21:57:52,860 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4256fff2fa33299a6264dd460a4003fb/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T21:57:52,874 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened 4256fff2fa33299a6264dd460a4003fb; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68684380, jitterRate=0.023477017879486084}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-15T21:57:52,875 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4256fff2fa33299a6264dd460a4003fb 2024-11-15T21:57:52,875 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for 4256fff2fa33299a6264dd460a4003fb: Running coprocessor pre-open hook at 1731707872281Writing region info on filesystem at 1731707872281Initializing all the Stores at 1731707872284 (+3 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707872284Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707872285 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707872285Obtaining lock to block concurrent updates at 1731707872329 (+44 ms)Preparing flush snapshotting stores in 4256fff2fa33299a6264dd460a4003fb at 1731707872329Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731707682333.4256fff2fa33299a6264dd460a4003fb., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731707872330 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731707682333.4256fff2fa33299a6264dd460a4003fb. at 1731707872330Flushing 4256fff2fa33299a6264dd460a4003fb/colfamily11: creating writer at 1731707872330Flushing 4256fff2fa33299a6264dd460a4003fb/colfamily11: appending metadata at 1731707872359 (+29 ms)Flushing 4256fff2fa33299a6264dd460a4003fb/colfamily11: closing flushed file at 1731707872360 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@369b3a77: reopening flushed file at 1731707872832 (+472 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 4256fff2fa33299a6264dd460a4003fb in 513ms, sequenceid=4, compaction requested=false; wal=null at 1731707872842 (+10 ms)Cleaning up temporary data from old regions at 1731707872844 (+2 ms)Running coprocessor post-open hooks at 1731707872875 (+31 ms)Region opened successfully at 1731707872875 2024-11-15T21:57:52,878 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731707682333.4256fff2fa33299a6264dd460a4003fb., pid=79, masterSystemTime=1731707871454 2024-11-15T21:57:52,886 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731707682333.4256fff2fa33299a6264dd460a4003fb. 2024-11-15T21:57:52,886 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731707682333.4256fff2fa33299a6264dd460a4003fb. 2024-11-15T21:57:52,890 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=4256fff2fa33299a6264dd460a4003fb, regionState=OPEN, openSeqNum=5, regionLocation=ae3311ea5ccb,40941,1731707679516 2024-11-15T21:57:52,898 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4256fff2fa33299a6264dd460a4003fb, server=ae3311ea5ccb,40941,1731707679516 because future has completed 2024-11-15T21:57:52,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=68 2024-11-15T21:57:52,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=68, state=SUCCESS, hasLock=false; OpenRegionProcedure 4256fff2fa33299a6264dd460a4003fb, server=ae3311ea5ccb,40941,1731707679516 in 1.5640 sec 2024-11-15T21:57:52,922 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=68, resume processing ppid=57 2024-11-15T21:57:52,922 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4256fff2fa33299a6264dd460a4003fb, ASSIGN in 1.7840 sec 2024-11-15T21:57:52,923 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server ae3311ea5ccb,32871,1731707679650 after splitting done 2024-11-15T21:57:52,923 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-15T21:57:52,923 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=SUCCESS, hasLock=true; ServerCrashProcedure ae3311ea5ccb,32871,1731707679650, splitWal=true, meta=true 2024-11-15T21:57:52,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T21:57:52,924 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=SUCCESS, hasLock=true; ServerCrashProcedure ae3311ea5ccb,32871,1731707679650, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T21:57:53,135 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 8 exec state=pid=57, state=SUCCESS, hasLock=true; ServerCrashProcedure ae3311ea5ccb,32871,1731707679650, splitWal=true, meta=true 2024-11-15T21:57:53,135 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T21:57:53,135 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T21:57:53,135 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T21:57:53,136 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T21:57:53,136 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T21:57:53,136 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T21:57:53,136 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T21:57:53,136 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T21:57:53,137 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T21:57:53,139 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T21:57:53,139 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T21:57:53,140 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T21:57:53,140 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T21:57:53,140 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T21:57:53,140 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T21:57:53,140 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T21:57:53,140 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T21:57:53,140 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T21:57:53,140 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T21:57:53,140 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T21:57:53,140 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T21:57:53,140 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T21:57:53,140 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T21:57:53,140 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T21:57:53,140 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T21:57:53,140 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T21:57:53,140 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T21:57:53,140 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T21:57:53,140 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T21:57:53,140 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T21:57:53,140 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T21:57:53,140 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T21:57:53,140 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T21:57:53,140 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T21:57:53,141 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T21:57:53,141 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-15T21:57:53,145 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_FINISH, hasLock=false; ServerCrashProcedure ae3311ea5ccb,32871,1731707679650, splitWal=true, meta=true 2024-11-15T21:57:53,145 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ae3311ea5ccb%2C32871%2C1731707679650.meta.1731707681806.meta, worker=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:57:53,145 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ae3311ea5ccb%2C32871%2C1731707679650.meta.1731707681806.meta, worker=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:57:53,145 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T21:57:53,145 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ae3311ea5ccb,40941,1731707679516 2024-11-15T21:57:53,145 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ae3311ea5ccb%2C32871%2C1731707679650.1731707681383, worker=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:57:53,145 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ae3311ea5ccb%2C32871%2C1731707679650.1731707681383, worker=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:57:53,145 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=dd723478fcfc0e25b606965c9135244b, ASSIGN 2024-11-15T21:57:53,145 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=aec2e46cb64898ba2a66825d3732f6dc, ASSIGN 2024-11-15T21:57:53,145 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=f9ef867a298d175eca9d85304f2faa69, ASSIGN 2024-11-15T21:57:53,145 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3cf264afea186a5cc96edd3e68a10d45, ASSIGN 2024-11-15T21:57:53,145 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4256fff2fa33299a6264dd460a4003fb, ASSIGN 2024-11-15T21:57:53,145 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d9989b0341ef96818e811f874dbcf91c, ASSIGN 2024-11-15T21:57:53,145 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=6e3b749fcbfdadfe59f1d1cf94f5406a, ASSIGN 2024-11-15T21:57:53,145 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d6063c53955b74f03cc36bd578a6bd41, ASSIGN 2024-11-15T21:57:53,145 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=6694325f9c862a5a1ee02eabf7861cc3, ASSIGN 2024-11-15T21:57:53,145 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=67, state=SUCCESS, hasLock=false; OpenRegionProcedure 3cf264afea186a5cc96edd3e68a10d45, server=ae3311ea5ccb,40941,1731707679516 2024-11-15T21:57:53,145 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=70, state=SUCCESS, hasLock=false; OpenRegionProcedure 6e3b749fcbfdadfe59f1d1cf94f5406a, server=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:57:53,146 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure dd723478fcfc0e25b606965c9135244b, server=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:57:53,146 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=66, state=SUCCESS, hasLock=false; OpenRegionProcedure f9ef867a298d175eca9d85304f2faa69, server=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:57:53,146 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=69, state=SUCCESS, hasLock=false; OpenRegionProcedure d9989b0341ef96818e811f874dbcf91c, server=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:57:53,146 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=72, state=SUCCESS, hasLock=false; OpenRegionProcedure 6694325f9c862a5a1ee02eabf7861cc3, server=ae3311ea5ccb,37455,1731707679725 2024-11-15T21:57:53,146 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=68, state=SUCCESS, hasLock=false; OpenRegionProcedure 4256fff2fa33299a6264dd460a4003fb, server=ae3311ea5ccb,40941,1731707679516 2024-11-15T21:57:53,146 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=71, state=SUCCESS, hasLock=false; OpenRegionProcedure d6063c53955b74f03cc36bd578a6bd41, server=ae3311ea5ccb,40941,1731707679516 2024-11-15T21:57:53,146 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=65, state=SUCCESS, hasLock=false; OpenRegionProcedure aec2e46cb64898ba2a66825d3732f6dc, server=ae3311ea5ccb,40941,1731707679516 2024-11-15T21:57:53,146 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_FINISH, hasLock=false; ServerCrashProcedure ae3311ea5ccb,32871,1731707679650, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T21:57:53,146 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ae3311ea5ccb%2C32871%2C1731707679650.meta.1731707681806.meta, worker=ae3311ea5ccb,37455,1731707679725 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T21:57:53,146 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ae3311ea5ccb%2C32871%2C1731707679650.meta.1731707681806.meta, worker=ae3311ea5ccb,37455,1731707679725 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T21:57:53,146 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T21:57:53,146 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ae3311ea5ccb,40941,1731707679516 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T21:57:53,146 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ae3311ea5ccb%2C32871%2C1731707679650.1731707681383, worker=ae3311ea5ccb,37455,1731707679725 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T21:57:53,146 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ae3311ea5ccb%2C32871%2C1731707679650.1731707681383, worker=ae3311ea5ccb,37455,1731707679725 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T21:57:53,146 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=dd723478fcfc0e25b606965c9135244b, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T21:57:53,146 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=aec2e46cb64898ba2a66825d3732f6dc, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T21:57:53,146 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=f9ef867a298d175eca9d85304f2faa69, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T21:57:53,146 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=67, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3cf264afea186a5cc96edd3e68a10d45, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T21:57:53,146 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=68, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4256fff2fa33299a6264dd460a4003fb, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T21:57:53,146 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=69, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d9989b0341ef96818e811f874dbcf91c, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T21:57:53,146 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=70, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=6e3b749fcbfdadfe59f1d1cf94f5406a, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T21:57:53,146 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=71, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d6063c53955b74f03cc36bd578a6bd41, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T21:57:53,146 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=6694325f9c862a5a1ee02eabf7861cc3, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T21:57:53,146 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=67, state=SUCCESS, hasLock=false; OpenRegionProcedure 3cf264afea186a5cc96edd3e68a10d45, server=ae3311ea5ccb,40941,1731707679516 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T21:57:53,146 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=70, state=SUCCESS, hasLock=false; OpenRegionProcedure 6e3b749fcbfdadfe59f1d1cf94f5406a, server=ae3311ea5ccb,37455,1731707679725 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T21:57:53,146 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure dd723478fcfc0e25b606965c9135244b, server=ae3311ea5ccb,37455,1731707679725 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T21:57:53,146 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=66, state=SUCCESS, hasLock=false; OpenRegionProcedure f9ef867a298d175eca9d85304f2faa69, server=ae3311ea5ccb,37455,1731707679725 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T21:57:53,146 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=69, state=SUCCESS, hasLock=false; OpenRegionProcedure d9989b0341ef96818e811f874dbcf91c, server=ae3311ea5ccb,37455,1731707679725 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T21:57:53,146 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=72, state=SUCCESS, hasLock=false; OpenRegionProcedure 6694325f9c862a5a1ee02eabf7861cc3, server=ae3311ea5ccb,37455,1731707679725 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T21:57:53,146 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=68, state=SUCCESS, hasLock=false; OpenRegionProcedure 4256fff2fa33299a6264dd460a4003fb, server=ae3311ea5ccb,40941,1731707679516 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T21:57:53,146 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=71, state=SUCCESS, hasLock=false; OpenRegionProcedure d6063c53955b74f03cc36bd578a6bd41, server=ae3311ea5ccb,40941,1731707679516 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T21:57:53,146 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=65, state=SUCCESS, hasLock=false; OpenRegionProcedure aec2e46cb64898ba2a66825d3732f6dc, server=ae3311ea5ccb,40941,1731707679516 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T21:57:53,147 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-15T21:57:53,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=61 2024-11-15T21:57:53,148 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ae3311ea5ccb,40941,1731707679516, table=hbase:meta, region=1588230740 2024-11-15T21:57:53,149 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ae3311ea5ccb,40941,1731707679516, state=OPEN 2024-11-15T21:57:53,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37455-0x100f86521fe0003, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T21:57:53,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40941-0x100f86521fe0001, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T21:57:53,155 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36531-0x100f86521fe0000, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T21:57:53,155 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T21:57:53,155 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T21:57:53,155 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T21:57:53,157 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731707903157 2024-11-15T21:57:53,157 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-15T21:57:53,167 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=229a036fd38ae13e7359b15bc2f2e587, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=2 2024-11-15T21:57:53,167 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dd723478fcfc0e25b606965c9135244b, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=5 2024-11-15T21:57:53,167 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50df10b3ae47c5dc7c4521c4c58c192f, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=2 2024-11-15T21:57:53,167 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4256fff2fa33299a6264dd460a4003fb, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=5 2024-11-15T21:57:53,168 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2c83200b2c422973a68afce1173aae30, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=2 2024-11-15T21:57:53,168 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f13d1331edcf64c426c3bf9433330e4, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=2 2024-11-15T21:57:53,168 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9ef867a298d175eca9d85304f2faa69, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=5 2024-11-15T21:57:53,168 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0890aeec1f0ca0649e6483a2d56f36b4, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=2 2024-11-15T21:57:53,168 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d28ee8a2ac06d2199c72ac7f884ba470, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=2 2024-11-15T21:57:53,168 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aec2e46cb64898ba2a66825d3732f6dc, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=5 2024-11-15T21:57:53,168 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cbb6b5b8341f7ff2ada4ac1931a1a588, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=2 2024-11-15T21:57:53,168 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a3f7be24d811f4d6177c28e08f25af2d, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=2 2024-11-15T21:57:53,168 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e3b749fcbfdadfe59f1d1cf94f5406a, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=5 2024-11-15T21:57:53,168 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51af260ba7a9b40a7f788a5c93a416b6, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=2 2024-11-15T21:57:53,168 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ab2d104e772f41ee9c56851adad43f0, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=2 2024-11-15T21:57:53,169 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3cf264afea186a5cc96edd3e68a10d45, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=5 2024-11-15T21:57:53,169 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d100add17d87270d000a3c39b49f8227, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=2 2024-11-15T21:57:53,169 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d9989b0341ef96818e811f874dbcf91c, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=5 2024-11-15T21:57:53,169 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=386f4af1e738f142f3fbde37773d4d2c, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=2 2024-11-15T21:57:53,169 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d6063c53955b74f03cc36bd578a6bd41, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=5 2024-11-15T21:57:53,169 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fac6af87aa0fc9826df0515f766c031e, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=2 2024-11-15T21:57:53,169 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ce283f02f0974e39c98759bb2bc9883, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=2 2024-11-15T21:57:53,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6694325f9c862a5a1ee02eabf7861cc3, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=5 2024-11-15T21:57:53,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46b5f83abe9585829b26884a1b26b43f, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=2 2024-11-15T21:57:53,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1031049c0b4149d1189db33e38ab56cd, regionState=OPEN, lastHost=ae3311ea5ccb,37455,1731707679725, regionLocation=ae3311ea5ccb,37455,1731707679725, openSeqNum=2 2024-11-15T21:57:53,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e76ae188febdca91d319999f619d9e11, regionState=OPEN, lastHost=ae3311ea5ccb,40941,1731707679516, regionLocation=ae3311ea5ccb,40941,1731707679516, openSeqNum=2 2024-11-15T21:57:53,170 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-15T21:57:53,170 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731707933170 2024-11-15T21:57:53,170 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731707993170 2024-11-15T21:57:53,170 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 12 msec 2024-11-15T21:57:53,170 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server ae3311ea5ccb,32871,1731707679650 after splitting done 2024-11-15T21:57:53,170 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-15T21:57:53,177 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; ServerCrashProcedure ae3311ea5ccb,32871,1731707679650, splitWal=true, meta=true in 3 mins, 5.327 sec 2024-11-15T21:57:53,429 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36066, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T21:57:53,434 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48846, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T21:57:53,447 WARN [RPCClient-NioEventLoopGroup-6-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server ae3311ea5ccb:32871 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: ae3311ea5ccb/172.17.0.3:32871 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T21:57:53,449 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731707682333.dd723478fcfc0e25b606965c9135244b., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731707682333.dd723478fcfc0e25b606965c9135244b., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2, error=java.net.ConnectException: Call to address=ae3311ea5ccb:32871 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:57:53,450 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731707682333.dd723478fcfc0e25b606965c9135244b., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2 is java.net.ConnectException: Connection refused 2024-11-15T21:57:53,450 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731707682333.dd723478fcfc0e25b606965c9135244b., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2 from cache 2024-11-15T21:57:53,450 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.FailedServers(52): Added failed server with address ae3311ea5ccb:32871 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: ae3311ea5ccb/172.17.0.3:32871 2024-11-15T21:57:53,555 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to ae3311ea5ccb:32871 this server is in the failed servers list 2024-11-15T21:57:53,556 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=ae3311ea5ccb:32871 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:57:53,556 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:57:53,556 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32871,1731707679650, seqNum=-1 from cache 2024-11-15T21:57:53,664 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T21:57:53,670 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36082, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T21:57:53,671 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,40941,1731707679516, seqNum=-1] 2024-11-15T21:57:53,683 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='bbb', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731707682333.dd723478fcfc0e25b606965c9135244b., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=5] 2024-11-15T21:57:53,689 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35792, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T21:57:53,713 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.AbstractRpcClient(357): Not trying to connect to ae3311ea5ccb:32871 this server is in the failed servers list 2024-11-15T21:57:53,713 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731707682333.4256fff2fa33299a6264dd460a4003fb., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731707682333.4256fff2fa33299a6264dd460a4003fb., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=ae3311ea5ccb:32871 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:57:53,714 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731707682333.4256fff2fa33299a6264dd460a4003fb., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:57:53,714 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731707682333.4256fff2fa33299a6264dd460a4003fb., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2 from cache 2024-11-15T21:57:53,830 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddd', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731707682333.4256fff2fa33299a6264dd460a4003fb., hostname=ae3311ea5ccb,40941,1731707679516, seqNum=5] 2024-11-15T21:57:53,854 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to ae3311ea5ccb:32871 this server is in the failed servers list 2024-11-15T21:57:53,855 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731707682333.f9ef867a298d175eca9d85304f2faa69., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731707682333.f9ef867a298d175eca9d85304f2faa69., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=ae3311ea5ccb:32871 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:57:53,855 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731707682333.f9ef867a298d175eca9d85304f2faa69., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:57:53,856 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731707682333.f9ef867a298d175eca9d85304f2faa69., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2 from cache 2024-11-15T21:57:53,950 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T21:57:53,973 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731707682333.f9ef867a298d175eca9d85304f2faa69., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=5] 2024-11-15T21:57:54,009 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to ae3311ea5ccb:32871 this server is in the failed servers list 2024-11-15T21:57:54,010 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731707682333.aec2e46cb64898ba2a66825d3732f6dc., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731707682333.aec2e46cb64898ba2a66825d3732f6dc., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=ae3311ea5ccb:32871 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:57:54,010 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731707682333.aec2e46cb64898ba2a66825d3732f6dc., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:57:54,010 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731707682333.aec2e46cb64898ba2a66825d3732f6dc., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2 from cache 2024-11-15T21:57:54,127 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjj', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731707682333.aec2e46cb64898ba2a66825d3732f6dc., hostname=ae3311ea5ccb,40941,1731707679516, seqNum=5] 2024-11-15T21:57:54,161 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to ae3311ea5ccb:32871 this server is in the failed servers list 2024-11-15T21:57:54,162 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731707682333.6e3b749fcbfdadfe59f1d1cf94f5406a., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731707682333.6e3b749fcbfdadfe59f1d1cf94f5406a., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=ae3311ea5ccb:32871 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:57:54,162 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731707682333.6e3b749fcbfdadfe59f1d1cf94f5406a., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:57:54,162 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731707682333.6e3b749fcbfdadfe59f1d1cf94f5406a., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2 from cache 2024-11-15T21:57:54,280 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731707682333.6e3b749fcbfdadfe59f1d1cf94f5406a., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=5] 2024-11-15T21:57:54,307 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to ae3311ea5ccb:32871 this server is in the failed servers list 2024-11-15T21:57:54,308 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731707682333.3cf264afea186a5cc96edd3e68a10d45., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731707682333.3cf264afea186a5cc96edd3e68a10d45., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=ae3311ea5ccb:32871 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:57:54,308 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731707682333.3cf264afea186a5cc96edd3e68a10d45., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:57:54,308 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731707682333.3cf264afea186a5cc96edd3e68a10d45., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2 from cache 2024-11-15T21:57:54,423 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731707682333.3cf264afea186a5cc96edd3e68a10d45., hostname=ae3311ea5ccb,40941,1731707679516, seqNum=5] 2024-11-15T21:57:54,444 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.AbstractRpcClient(357): Not trying to connect to ae3311ea5ccb:32871 this server is in the failed servers list 2024-11-15T21:57:54,444 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731707682333.d9989b0341ef96818e811f874dbcf91c., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731707682333.d9989b0341ef96818e811f874dbcf91c., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=ae3311ea5ccb:32871 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:57:54,445 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731707682333.d9989b0341ef96818e811f874dbcf91c., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:57:54,445 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731707682333.d9989b0341ef96818e811f874dbcf91c., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2 from cache 2024-11-15T21:57:54,563 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rrr', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731707682333.d9989b0341ef96818e811f874dbcf91c., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=5] 2024-11-15T21:57:54,581 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to ae3311ea5ccb:32871 this server is in the failed servers list 2024-11-15T21:57:54,582 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731707682333.d6063c53955b74f03cc36bd578a6bd41., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731707682333.d6063c53955b74f03cc36bd578a6bd41., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=ae3311ea5ccb:32871 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:57:54,582 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731707682333.d6063c53955b74f03cc36bd578a6bd41., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:57:54,582 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731707682333.d6063c53955b74f03cc36bd578a6bd41., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2 from cache 2024-11-15T21:57:54,691 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ttt', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731707682333.d6063c53955b74f03cc36bd578a6bd41., hostname=ae3311ea5ccb,40941,1731707679516, seqNum=5] 2024-11-15T21:57:54,710 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to ae3311ea5ccb:32871 this server is in the failed servers list 2024-11-15T21:57:54,710 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731707682333.6694325f9c862a5a1ee02eabf7861cc3., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731707682333.6694325f9c862a5a1ee02eabf7861cc3., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=ae3311ea5ccb:32871 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:57:54,710 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731707682333.6694325f9c862a5a1ee02eabf7861cc3., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ae3311ea5ccb:32871 2024-11-15T21:57:54,710 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731707682333.6694325f9c862a5a1ee02eabf7861cc3., hostname=ae3311ea5ccb,32871,1731707679650, seqNum=2 from cache 2024-11-15T21:57:54,819 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='www', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731707682333.6694325f9c862a5a1ee02eabf7861cc3., hostname=ae3311ea5ccb,37455,1731707679725, seqNum=5] 2024-11-15T21:57:54,950 WARN [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false 2024-11-15T21:57:54,950 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-15T21:57:54,950 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T21:57:54,951 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.procedure.TestSCPBase.tearDown(TestSCPBase.java:72) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T21:57:54,951 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T21:57:54,951 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T21:57:54,951 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T21:57:54,951 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-15T21:57:54,952 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1030710501, stopped=false 2024-11-15T21:57:54,952 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=ae3311ea5ccb,36531,1731707678636 2024-11-15T21:57:54,962 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37455-0x100f86521fe0003, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-15T21:57:54,962 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37455-0x100f86521fe0003, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:57:54,962 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40941-0x100f86521fe0001, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-15T21:57:54,962 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40941-0x100f86521fe0001, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:57:54,963 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36531-0x100f86521fe0000, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-15T21:57:54,963 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T21:57:54,963 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T21:57:54,963 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.procedure.TestSCPBase.tearDown(TestSCPBase.java:72) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T21:57:54,963 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T21:57:54,963 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ae3311ea5ccb,40941,1731707679516' ***** 2024-11-15T21:57:54,964 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-15T21:57:54,964 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ae3311ea5ccb,37455,1731707679725' ***** 2024-11-15T21:57:54,964 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-15T21:57:54,963 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36531-0x100f86521fe0000, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:57:54,965 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36531-0x100f86521fe0000, quorum=127.0.0.1:58023, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T21:57:54,965 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40941-0x100f86521fe0001, quorum=127.0.0.1:58023, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T21:57:54,966 INFO [RS:2;ae3311ea5ccb:37455 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-15T21:57:54,966 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37455-0x100f86521fe0003, quorum=127.0.0.1:58023, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T21:57:54,966 INFO [RS:2;ae3311ea5ccb:37455 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-15T21:57:54,966 INFO [RS:2;ae3311ea5ccb:37455 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-15T21:57:54,966 INFO [RS:2;ae3311ea5ccb:37455 {}] regionserver.HRegionServer(3091): Received CLOSE for 51af260ba7a9b40a7f788a5c93a416b6 2024-11-15T21:57:54,967 INFO [RS:0;ae3311ea5ccb:40941 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-15T21:57:54,967 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-15T21:57:54,967 INFO [RS:0;ae3311ea5ccb:40941 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-15T21:57:54,967 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-15T21:57:54,967 INFO [RS:0;ae3311ea5ccb:40941 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-15T21:57:54,967 INFO [RS:0;ae3311ea5ccb:40941 {}] regionserver.HRegionServer(3091): Received CLOSE for e76ae188febdca91d319999f619d9e11 2024-11-15T21:57:54,977 INFO [RS:2;ae3311ea5ccb:37455 {}] regionserver.HRegionServer(3091): Received CLOSE for 6e3b749fcbfdadfe59f1d1cf94f5406a 2024-11-15T21:57:54,977 INFO [RS:2;ae3311ea5ccb:37455 {}] regionserver.HRegionServer(3091): Received CLOSE for cbb6b5b8341f7ff2ada4ac1931a1a588 2024-11-15T21:57:54,977 INFO [RS:2;ae3311ea5ccb:37455 {}] regionserver.HRegionServer(3091): Received CLOSE for d100add17d87270d000a3c39b49f8227 2024-11-15T21:57:54,977 INFO [RS:2;ae3311ea5ccb:37455 {}] regionserver.HRegionServer(3091): Received CLOSE for dd723478fcfc0e25b606965c9135244b 2024-11-15T21:57:54,977 INFO [RS:2;ae3311ea5ccb:37455 {}] regionserver.HRegionServer(3091): Received CLOSE for f9ef867a298d175eca9d85304f2faa69 2024-11-15T21:57:54,977 INFO [RS:2;ae3311ea5ccb:37455 {}] regionserver.HRegionServer(3091): Received CLOSE for fac6af87aa0fc9826df0515f766c031e 2024-11-15T21:57:54,977 INFO [RS:2;ae3311ea5ccb:37455 {}] regionserver.HRegionServer(3091): Received CLOSE for 2c83200b2c422973a68afce1173aae30 2024-11-15T21:57:54,977 INFO [RS:2;ae3311ea5ccb:37455 {}] regionserver.HRegionServer(3091): Received CLOSE for 6694325f9c862a5a1ee02eabf7861cc3 2024-11-15T21:57:54,977 INFO [RS:2;ae3311ea5ccb:37455 {}] regionserver.HRegionServer(3091): Received CLOSE for d9989b0341ef96818e811f874dbcf91c 2024-11-15T21:57:54,977 INFO [RS:2;ae3311ea5ccb:37455 {}] regionserver.HRegionServer(3091): Received CLOSE for 1031049c0b4149d1189db33e38ab56cd 2024-11-15T21:57:54,977 INFO [RS:2;ae3311ea5ccb:37455 {}] regionserver.HRegionServer(3091): Received CLOSE for 46b5f83abe9585829b26884a1b26b43f 2024-11-15T21:57:54,977 INFO [RS:2;ae3311ea5ccb:37455 {}] regionserver.HRegionServer(3091): Received CLOSE for 50df10b3ae47c5dc7c4521c4c58c192f 2024-11-15T21:57:54,977 INFO [RS:2;ae3311ea5ccb:37455 {}] regionserver.HRegionServer(3091): Received CLOSE for 0890aeec1f0ca0649e6483a2d56f36b4 2024-11-15T21:57:54,977 INFO [RS:2;ae3311ea5ccb:37455 {}] regionserver.HRegionServer(959): stopping server ae3311ea5ccb,37455,1731707679725 2024-11-15T21:57:54,977 INFO [RS:0;ae3311ea5ccb:40941 {}] regionserver.HRegionServer(3091): Received CLOSE for 0ab2d104e772f41ee9c56851adad43f0 2024-11-15T21:57:54,977 INFO [RS:2;ae3311ea5ccb:37455 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-15T21:57:54,977 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 51af260ba7a9b40a7f788a5c93a416b6, disabling compactions & flushes 2024-11-15T21:57:54,977 INFO [RS:0;ae3311ea5ccb:40941 {}] regionserver.HRegionServer(3091): Received CLOSE for a3f7be24d811f4d6177c28e08f25af2d 2024-11-15T21:57:54,977 INFO [RS:0;ae3311ea5ccb:40941 {}] regionserver.HRegionServer(3091): Received CLOSE for 229a036fd38ae13e7359b15bc2f2e587 2024-11-15T21:57:54,977 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731707682333.51af260ba7a9b40a7f788a5c93a416b6. 2024-11-15T21:57:54,977 INFO [RS:2;ae3311ea5ccb:37455 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;ae3311ea5ccb:37455. 2024-11-15T21:57:54,977 INFO [RS:0;ae3311ea5ccb:40941 {}] regionserver.HRegionServer(3091): Received CLOSE for 386f4af1e738f142f3fbde37773d4d2c 2024-11-15T21:57:54,977 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731707682333.51af260ba7a9b40a7f788a5c93a416b6. 2024-11-15T21:57:54,977 INFO [RS:0;ae3311ea5ccb:40941 {}] regionserver.HRegionServer(3091): Received CLOSE for d28ee8a2ac06d2199c72ac7f884ba470 2024-11-15T21:57:54,977 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731707682333.51af260ba7a9b40a7f788a5c93a416b6. after waiting 0 ms 2024-11-15T21:57:54,977 INFO [RS:0;ae3311ea5ccb:40941 {}] regionserver.HRegionServer(3091): Received CLOSE for 4256fff2fa33299a6264dd460a4003fb 2024-11-15T21:57:54,977 DEBUG [RS:2;ae3311ea5ccb:37455 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T21:57:54,977 INFO [RS:0;ae3311ea5ccb:40941 {}] regionserver.HRegionServer(3091): Received CLOSE for d6063c53955b74f03cc36bd578a6bd41 2024-11-15T21:57:54,977 DEBUG [RS:2;ae3311ea5ccb:37455 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T21:57:54,977 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731707682333.51af260ba7a9b40a7f788a5c93a416b6. 2024-11-15T21:57:54,977 INFO [RS:0;ae3311ea5ccb:40941 {}] regionserver.HRegionServer(3091): Received CLOSE for aec2e46cb64898ba2a66825d3732f6dc 2024-11-15T21:57:54,977 INFO [RS:0;ae3311ea5ccb:40941 {}] regionserver.HRegionServer(3091): Received CLOSE for 3cf264afea186a5cc96edd3e68a10d45 2024-11-15T21:57:54,978 INFO [RS:0;ae3311ea5ccb:40941 {}] regionserver.HRegionServer(3091): Received CLOSE for 5ce283f02f0974e39c98759bb2bc9883 2024-11-15T21:57:54,978 INFO [RS:0;ae3311ea5ccb:40941 {}] regionserver.HRegionServer(3091): Received CLOSE for 3f13d1331edcf64c426c3bf9433330e4 2024-11-15T21:57:54,978 INFO [RS:0;ae3311ea5ccb:40941 {}] regionserver.HRegionServer(959): stopping server ae3311ea5ccb,40941,1731707679516 2024-11-15T21:57:54,978 INFO [RS:0;ae3311ea5ccb:40941 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-15T21:57:54,978 INFO [RS:0;ae3311ea5ccb:40941 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;ae3311ea5ccb:40941. 2024-11-15T21:57:54,978 INFO [RS:2;ae3311ea5ccb:37455 {}] regionserver.HRegionServer(1321): Waiting on 14 regions to close 2024-11-15T21:57:54,978 DEBUG [RS:0;ae3311ea5ccb:40941 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T21:57:54,978 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 51af260ba7a9b40a7f788a5c93a416b6 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-15T21:57:54,978 DEBUG [RS:0;ae3311ea5ccb:40941 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T21:57:54,978 INFO [RS:0;ae3311ea5ccb:40941 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-15T21:57:54,978 INFO [RS:0;ae3311ea5ccb:40941 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-15T21:57:54,978 INFO [RS:0;ae3311ea5ccb:40941 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-15T21:57:54,978 DEBUG [RS:2;ae3311ea5ccb:37455 {}] regionserver.HRegionServer(1351): Waiting on 0890aeec1f0ca0649e6483a2d56f36b4, 1031049c0b4149d1189db33e38ab56cd, 2c83200b2c422973a68afce1173aae30, 46b5f83abe9585829b26884a1b26b43f, 50df10b3ae47c5dc7c4521c4c58c192f, 51af260ba7a9b40a7f788a5c93a416b6, 6694325f9c862a5a1ee02eabf7861cc3, 6e3b749fcbfdadfe59f1d1cf94f5406a, cbb6b5b8341f7ff2ada4ac1931a1a588, d100add17d87270d000a3c39b49f8227, d9989b0341ef96818e811f874dbcf91c, dd723478fcfc0e25b606965c9135244b, f9ef867a298d175eca9d85304f2faa69, fac6af87aa0fc9826df0515f766c031e 2024-11-15T21:57:54,978 INFO [RS:0;ae3311ea5ccb:40941 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-15T21:57:54,981 INFO [RS:0;ae3311ea5ccb:40941 {}] regionserver.HRegionServer(1321): Waiting on 13 regions to close 2024-11-15T21:57:54,981 DEBUG [RS:0;ae3311ea5ccb:40941 {}] regionserver.HRegionServer(1351): Waiting on 0ab2d104e772f41ee9c56851adad43f0, 1588230740, 229a036fd38ae13e7359b15bc2f2e587, 386f4af1e738f142f3fbde37773d4d2c, 3cf264afea186a5cc96edd3e68a10d45, 3f13d1331edcf64c426c3bf9433330e4, 4256fff2fa33299a6264dd460a4003fb, 5ce283f02f0974e39c98759bb2bc9883, a3f7be24d811f4d6177c28e08f25af2d, aec2e46cb64898ba2a66825d3732f6dc, d28ee8a2ac06d2199c72ac7f884ba470, d6063c53955b74f03cc36bd578a6bd41, e76ae188febdca91d319999f619d9e11 2024-11-15T21:57:54,981 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e76ae188febdca91d319999f619d9e11, disabling compactions & flushes 2024-11-15T21:57:54,981 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731707682333.e76ae188febdca91d319999f619d9e11. 2024-11-15T21:57:54,981 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731707682333.e76ae188febdca91d319999f619d9e11. 2024-11-15T21:57:54,981 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731707682333.e76ae188febdca91d319999f619d9e11. after waiting 0 ms 2024-11-15T21:57:54,981 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731707682333.e76ae188febdca91d319999f619d9e11. 2024-11-15T21:57:54,981 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing e76ae188febdca91d319999f619d9e11 3/3 column families, dataSize=48 B heapSize=888 B 2024-11-15T21:57:54,985 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-15T21:57:54,985 INFO [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-15T21:57:54,985 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-15T21:57:54,985 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-15T21:57:54,985 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-15T21:57:54,986 INFO [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=13.39 KB heapSize=19.70 KB 2024-11-15T21:57:54,997 INFO [regionserver/ae3311ea5ccb:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T21:57:55,000 INFO [regionserver/ae3311ea5ccb:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T21:57:55,017 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/hbase/meta/1588230740/.tmp/info/c4598888e5474fa09fbb17f9ea017321 is 268, key is testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731707682333.dd723478fcfc0e25b606965c9135244b./info:regioninfo/1731707872232/Put/seqid=0 2024-11-15T21:57:55,035 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/51af260ba7a9b40a7f788a5c93a416b6/.tmp/colfamily11/b852e726bae7454c9dd2f0595f2b5fe0 is 52, key is nnn/colfamily11:colfamily11/1731707686754/Put/seqid=0 2024-11-15T21:57:55,037 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e76ae188febdca91d319999f619d9e11/.tmp/colfamily11/2be088a0415f495b8b2cc68ca1f1481e is 52, key is zzz/colfamily11:colfamily11/1731707686825/Put/seqid=0 2024-11-15T21:57:55,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741887_1063 (size=19858) 2024-11-15T21:57:55,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741887_1063 (size=19858) 2024-11-15T21:57:55,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741887_1063 (size=19858) 2024-11-15T21:57:55,139 INFO [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.39 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/hbase/meta/1588230740/.tmp/info/c4598888e5474fa09fbb17f9ea017321 2024-11-15T21:57:55,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741888_1064 (size=39263) 2024-11-15T21:57:55,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741888_1064 (size=39263) 2024-11-15T21:57:55,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741888_1064 (size=39263) 2024-11-15T21:57:55,176 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/hbase/meta/1588230740/.tmp/info/c4598888e5474fa09fbb17f9ea017321 as hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/hbase/meta/1588230740/info/c4598888e5474fa09fbb17f9ea017321 2024-11-15T21:57:55,178 DEBUG [RS:2;ae3311ea5ccb:37455 {}] regionserver.HRegionServer(1351): Waiting on 0890aeec1f0ca0649e6483a2d56f36b4, 1031049c0b4149d1189db33e38ab56cd, 2c83200b2c422973a68afce1173aae30, 46b5f83abe9585829b26884a1b26b43f, 50df10b3ae47c5dc7c4521c4c58c192f, 51af260ba7a9b40a7f788a5c93a416b6, 6694325f9c862a5a1ee02eabf7861cc3, 6e3b749fcbfdadfe59f1d1cf94f5406a, cbb6b5b8341f7ff2ada4ac1931a1a588, d100add17d87270d000a3c39b49f8227, d9989b0341ef96818e811f874dbcf91c, dd723478fcfc0e25b606965c9135244b, f9ef867a298d175eca9d85304f2faa69, fac6af87aa0fc9826df0515f766c031e 2024-11-15T21:57:55,181 DEBUG [RS:0;ae3311ea5ccb:40941 {}] regionserver.HRegionServer(1351): Waiting on 0ab2d104e772f41ee9c56851adad43f0, 1588230740, 229a036fd38ae13e7359b15bc2f2e587, 386f4af1e738f142f3fbde37773d4d2c, 3cf264afea186a5cc96edd3e68a10d45, 3f13d1331edcf64c426c3bf9433330e4, 4256fff2fa33299a6264dd460a4003fb, 5ce283f02f0974e39c98759bb2bc9883, a3f7be24d811f4d6177c28e08f25af2d, aec2e46cb64898ba2a66825d3732f6dc, d28ee8a2ac06d2199c72ac7f884ba470, d6063c53955b74f03cc36bd578a6bd41, e76ae188febdca91d319999f619d9e11 2024-11-15T21:57:55,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741889_1065 (size=4857) 2024-11-15T21:57:55,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741889_1065 (size=4857) 2024-11-15T21:57:55,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741889_1065 (size=4857) 2024-11-15T21:57:55,186 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e76ae188febdca91d319999f619d9e11/.tmp/colfamily11/2be088a0415f495b8b2cc68ca1f1481e 2024-11-15T21:57:55,188 INFO [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/hbase/meta/1588230740/info/c4598888e5474fa09fbb17f9ea017321, entries=72, sequenceid=81, filesize=19.4 K 2024-11-15T21:57:55,193 INFO [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~13.39 KB/13716, heapSize ~18.94 KB/19392, currentSize=0 B/0 for 1588230740 in 208ms, sequenceid=81, compaction requested=false 2024-11-15T21:57:55,194 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e76ae188febdca91d319999f619d9e11/.tmp/colfamily11/2be088a0415f495b8b2cc68ca1f1481e as hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e76ae188febdca91d319999f619d9e11/colfamily11/2be088a0415f495b8b2cc68ca1f1481e 2024-11-15T21:57:55,204 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e76ae188febdca91d319999f619d9e11/colfamily11/2be088a0415f495b8b2cc68ca1f1481e, entries=1, sequenceid=5, filesize=4.7 K 2024-11-15T21:57:55,207 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~48 B/48, heapSize ~360 B/360, currentSize=0 B/0 for e76ae188febdca91d319999f619d9e11 in 226ms, sequenceid=5, compaction requested=false 2024-11-15T21:57:55,213 INFO [regionserver/ae3311ea5ccb:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-15T21:57:55,213 INFO [regionserver/ae3311ea5ccb:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-15T21:57:55,217 INFO [regionserver/ae3311ea5ccb:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-15T21:57:55,217 INFO [regionserver/ae3311ea5ccb:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-15T21:57:55,242 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/hbase/meta/1588230740/recovered.edits/84.seqid, newMaxSeqId=84, maxSeqId=60 2024-11-15T21:57:55,245 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e76ae188febdca91d319999f619d9e11/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-15T21:57:55,250 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-15T21:57:55,250 INFO [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-15T21:57:55,251 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731707874985Running coprocessor pre-close hooks at 1731707874985Disabling compacts and flushes for region at 1731707874985Disabling writes for close at 1731707874985Obtaining lock to block concurrent updates at 1731707874986 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731707874986Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=13716, getHeapSize=20112, getOffHeapSize=0, getCellsCount=72 at 1731707874986Flushing stores of hbase:meta,,1.1588230740 at 1731707874994 (+8 ms)Flushing 1588230740/info: creating writer at 1731707874994Flushing 1588230740/info: appending metadata at 1731707875016 (+22 ms)Flushing 1588230740/info: closing flushed file at 1731707875016Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@52c7ce28: reopening flushed file at 1731707875168 (+152 ms)Finished flush of dataSize ~13.39 KB/13716, heapSize ~18.94 KB/19392, currentSize=0 B/0 for 1588230740 in 208ms, sequenceid=81, compaction requested=false at 1731707875193 (+25 ms)Writing region close event to WAL at 1731707875232 (+39 ms)Running coprocessor post-close hooks at 1731707875243 (+11 ms)Closed at 1731707875250 (+7 ms) 2024-11-15T21:57:55,251 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731707682333.e76ae188febdca91d319999f619d9e11. 2024-11-15T21:57:55,251 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e76ae188febdca91d319999f619d9e11: Waiting for close lock at 1731707874981Running coprocessor pre-close hooks at 1731707874981Disabling compacts and flushes for region at 1731707874981Disabling writes for close at 1731707874981Obtaining lock to block concurrent updates at 1731707874981Preparing flush snapshotting stores in e76ae188febdca91d319999f619d9e11 at 1731707874981Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731707682333.e76ae188febdca91d319999f619d9e11., syncing WAL and waiting on mvcc, flushsize=dataSize=48, getHeapSize=840, getOffHeapSize=0, getCellsCount=1 at 1731707874989 (+8 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731707682333.e76ae188febdca91d319999f619d9e11. at 1731707874991 (+2 ms)Flushing e76ae188febdca91d319999f619d9e11/colfamily11: creating writer at 1731707874991Flushing e76ae188febdca91d319999f619d9e11/colfamily11: appending metadata at 1731707875037 (+46 ms)Flushing e76ae188febdca91d319999f619d9e11/colfamily11: closing flushed file at 1731707875037Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d1b43cc: reopening flushed file at 1731707875193 (+156 ms)Finished flush of dataSize ~48 B/48, heapSize ~360 B/360, currentSize=0 B/0 for e76ae188febdca91d319999f619d9e11 in 226ms, sequenceid=5, compaction requested=false at 1731707875208 (+15 ms)Writing region close event to WAL at 1731707875229 (+21 ms)Running coprocessor post-close hooks at 1731707875251 (+22 ms)Closed at 1731707875251 2024-11-15T21:57:55,252 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-15T21:57:55,253 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731707682333.e76ae188febdca91d319999f619d9e11. 2024-11-15T21:57:55,253 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0ab2d104e772f41ee9c56851adad43f0, disabling compactions & flushes 2024-11-15T21:57:55,253 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731707682333.0ab2d104e772f41ee9c56851adad43f0. 2024-11-15T21:57:55,253 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731707682333.0ab2d104e772f41ee9c56851adad43f0. 2024-11-15T21:57:55,253 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731707682333.0ab2d104e772f41ee9c56851adad43f0. after waiting 0 ms 2024-11-15T21:57:55,253 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731707682333.0ab2d104e772f41ee9c56851adad43f0. 2024-11-15T21:57:55,253 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 0ab2d104e772f41ee9c56851adad43f0 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-15T21:57:55,287 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0ab2d104e772f41ee9c56851adad43f0/.tmp/colfamily11/7274e716884f4af3a04d1a404312f605 is 52, key is ooo/colfamily11:colfamily11/1731707686726/Put/seqid=0 2024-11-15T21:57:55,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741890_1066 (size=39263) 2024-11-15T21:57:55,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741890_1066 (size=39263) 2024-11-15T21:57:55,314 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0ab2d104e772f41ee9c56851adad43f0/.tmp/colfamily11/7274e716884f4af3a04d1a404312f605 2024-11-15T21:57:55,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741890_1066 (size=39263) 2024-11-15T21:57:55,332 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0ab2d104e772f41ee9c56851adad43f0/.tmp/colfamily11/7274e716884f4af3a04d1a404312f605 as hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0ab2d104e772f41ee9c56851adad43f0/colfamily11/7274e716884f4af3a04d1a404312f605 2024-11-15T21:57:55,359 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0ab2d104e772f41ee9c56851adad43f0/colfamily11/7274e716884f4af3a04d1a404312f605, entries=703, sequenceid=5, filesize=38.3 K 2024-11-15T21:57:55,365 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 0ab2d104e772f41ee9c56851adad43f0 in 112ms, sequenceid=5, compaction requested=false 2024-11-15T21:57:55,379 DEBUG [RS:2;ae3311ea5ccb:37455 {}] regionserver.HRegionServer(1351): Waiting on 0890aeec1f0ca0649e6483a2d56f36b4, 1031049c0b4149d1189db33e38ab56cd, 2c83200b2c422973a68afce1173aae30, 46b5f83abe9585829b26884a1b26b43f, 50df10b3ae47c5dc7c4521c4c58c192f, 51af260ba7a9b40a7f788a5c93a416b6, 6694325f9c862a5a1ee02eabf7861cc3, 6e3b749fcbfdadfe59f1d1cf94f5406a, cbb6b5b8341f7ff2ada4ac1931a1a588, d100add17d87270d000a3c39b49f8227, d9989b0341ef96818e811f874dbcf91c, dd723478fcfc0e25b606965c9135244b, f9ef867a298d175eca9d85304f2faa69, fac6af87aa0fc9826df0515f766c031e 2024-11-15T21:57:55,385 DEBUG [RS:0;ae3311ea5ccb:40941 {}] regionserver.HRegionServer(1351): Waiting on 0ab2d104e772f41ee9c56851adad43f0, 229a036fd38ae13e7359b15bc2f2e587, 386f4af1e738f142f3fbde37773d4d2c, 3cf264afea186a5cc96edd3e68a10d45, 3f13d1331edcf64c426c3bf9433330e4, 4256fff2fa33299a6264dd460a4003fb, 5ce283f02f0974e39c98759bb2bc9883, a3f7be24d811f4d6177c28e08f25af2d, aec2e46cb64898ba2a66825d3732f6dc, d28ee8a2ac06d2199c72ac7f884ba470, d6063c53955b74f03cc36bd578a6bd41 2024-11-15T21:57:55,422 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0ab2d104e772f41ee9c56851adad43f0/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-15T21:57:55,431 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731707682333.0ab2d104e772f41ee9c56851adad43f0. 2024-11-15T21:57:55,431 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0ab2d104e772f41ee9c56851adad43f0: Waiting for close lock at 1731707875253Running coprocessor pre-close hooks at 1731707875253Disabling compacts and flushes for region at 1731707875253Disabling writes for close at 1731707875253Obtaining lock to block concurrent updates at 1731707875253Preparing flush snapshotting stores in 0ab2d104e772f41ee9c56851adad43f0 at 1731707875253Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731707682333.0ab2d104e772f41ee9c56851adad43f0., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731707875254 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731707682333.0ab2d104e772f41ee9c56851adad43f0. at 1731707875258 (+4 ms)Flushing 0ab2d104e772f41ee9c56851adad43f0/colfamily11: creating writer at 1731707875258Flushing 0ab2d104e772f41ee9c56851adad43f0/colfamily11: appending metadata at 1731707875287 (+29 ms)Flushing 0ab2d104e772f41ee9c56851adad43f0/colfamily11: closing flushed file at 1731707875287Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d1f1ebb: reopening flushed file at 1731707875331 (+44 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 0ab2d104e772f41ee9c56851adad43f0 in 112ms, sequenceid=5, compaction requested=false at 1731707875365 (+34 ms)Writing region close event to WAL at 1731707875381 (+16 ms)Running coprocessor post-close hooks at 1731707875430 (+49 ms)Closed at 1731707875431 (+1 ms) 2024-11-15T21:57:55,431 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731707682333.0ab2d104e772f41ee9c56851adad43f0. 2024-11-15T21:57:55,431 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a3f7be24d811f4d6177c28e08f25af2d, disabling compactions & flushes 2024-11-15T21:57:55,431 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731707682333.a3f7be24d811f4d6177c28e08f25af2d. 2024-11-15T21:57:55,431 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731707682333.a3f7be24d811f4d6177c28e08f25af2d. 2024-11-15T21:57:55,432 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731707682333.a3f7be24d811f4d6177c28e08f25af2d. after waiting 0 ms 2024-11-15T21:57:55,432 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731707682333.a3f7be24d811f4d6177c28e08f25af2d. 2024-11-15T21:57:55,432 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing a3f7be24d811f4d6177c28e08f25af2d 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-15T21:57:55,474 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a3f7be24d811f4d6177c28e08f25af2d/.tmp/colfamily11/7430ab34f90b468cb7b2ccdee973c857 is 52, key is lll/colfamily11:colfamily11/1731707686687/Put/seqid=0 2024-11-15T21:57:55,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741891_1067 (size=39263) 2024-11-15T21:57:55,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741891_1067 (size=39263) 2024-11-15T21:57:55,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741891_1067 (size=39263) 2024-11-15T21:57:55,544 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a3f7be24d811f4d6177c28e08f25af2d/.tmp/colfamily11/7430ab34f90b468cb7b2ccdee973c857 2024-11-15T21:57:55,562 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/51af260ba7a9b40a7f788a5c93a416b6/.tmp/colfamily11/b852e726bae7454c9dd2f0595f2b5fe0 2024-11-15T21:57:55,579 DEBUG [RS:2;ae3311ea5ccb:37455 {}] regionserver.HRegionServer(1351): Waiting on 0890aeec1f0ca0649e6483a2d56f36b4, 1031049c0b4149d1189db33e38ab56cd, 2c83200b2c422973a68afce1173aae30, 46b5f83abe9585829b26884a1b26b43f, 50df10b3ae47c5dc7c4521c4c58c192f, 51af260ba7a9b40a7f788a5c93a416b6, 6694325f9c862a5a1ee02eabf7861cc3, 6e3b749fcbfdadfe59f1d1cf94f5406a, cbb6b5b8341f7ff2ada4ac1931a1a588, d100add17d87270d000a3c39b49f8227, d9989b0341ef96818e811f874dbcf91c, dd723478fcfc0e25b606965c9135244b, f9ef867a298d175eca9d85304f2faa69, fac6af87aa0fc9826df0515f766c031e 2024-11-15T21:57:55,586 DEBUG [RS:0;ae3311ea5ccb:40941 {}] regionserver.HRegionServer(1351): Waiting on 229a036fd38ae13e7359b15bc2f2e587, 386f4af1e738f142f3fbde37773d4d2c, 3cf264afea186a5cc96edd3e68a10d45, 3f13d1331edcf64c426c3bf9433330e4, 4256fff2fa33299a6264dd460a4003fb, 5ce283f02f0974e39c98759bb2bc9883, a3f7be24d811f4d6177c28e08f25af2d, aec2e46cb64898ba2a66825d3732f6dc, d28ee8a2ac06d2199c72ac7f884ba470, d6063c53955b74f03cc36bd578a6bd41 2024-11-15T21:57:55,609 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a3f7be24d811f4d6177c28e08f25af2d/.tmp/colfamily11/7430ab34f90b468cb7b2ccdee973c857 as hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a3f7be24d811f4d6177c28e08f25af2d/colfamily11/7430ab34f90b468cb7b2ccdee973c857 2024-11-15T21:57:55,616 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/51af260ba7a9b40a7f788a5c93a416b6/.tmp/colfamily11/b852e726bae7454c9dd2f0595f2b5fe0 as hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/51af260ba7a9b40a7f788a5c93a416b6/colfamily11/b852e726bae7454c9dd2f0595f2b5fe0 2024-11-15T21:57:55,620 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a3f7be24d811f4d6177c28e08f25af2d/colfamily11/7430ab34f90b468cb7b2ccdee973c857, entries=703, sequenceid=5, filesize=38.3 K 2024-11-15T21:57:55,622 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for a3f7be24d811f4d6177c28e08f25af2d in 190ms, sequenceid=5, compaction requested=false 2024-11-15T21:57:55,628 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/51af260ba7a9b40a7f788a5c93a416b6/colfamily11/b852e726bae7454c9dd2f0595f2b5fe0, entries=703, sequenceid=5, filesize=38.3 K 2024-11-15T21:57:55,630 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 51af260ba7a9b40a7f788a5c93a416b6 in 653ms, sequenceid=5, compaction requested=false 2024-11-15T21:57:55,656 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/51af260ba7a9b40a7f788a5c93a416b6/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-15T21:57:55,657 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731707682333.51af260ba7a9b40a7f788a5c93a416b6. 2024-11-15T21:57:55,657 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 51af260ba7a9b40a7f788a5c93a416b6: Waiting for close lock at 1731707874977Running coprocessor pre-close hooks at 1731707874977Disabling compacts and flushes for region at 1731707874977Disabling writes for close at 1731707874977Obtaining lock to block concurrent updates at 1731707874978 (+1 ms)Preparing flush snapshotting stores in 51af260ba7a9b40a7f788a5c93a416b6 at 1731707874978Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731707682333.51af260ba7a9b40a7f788a5c93a416b6., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731707874984 (+6 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731707682333.51af260ba7a9b40a7f788a5c93a416b6. at 1731707874988 (+4 ms)Flushing 51af260ba7a9b40a7f788a5c93a416b6/colfamily11: creating writer at 1731707874988Flushing 51af260ba7a9b40a7f788a5c93a416b6/colfamily11: appending metadata at 1731707875032 (+44 ms)Flushing 51af260ba7a9b40a7f788a5c93a416b6/colfamily11: closing flushed file at 1731707875033 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7db42a1a: reopening flushed file at 1731707875610 (+577 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 51af260ba7a9b40a7f788a5c93a416b6 in 653ms, sequenceid=5, compaction requested=false at 1731707875631 (+21 ms)Writing region close event to WAL at 1731707875647 (+16 ms)Running coprocessor post-close hooks at 1731707875657 (+10 ms)Closed at 1731707875657 2024-11-15T21:57:55,657 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731707682333.51af260ba7a9b40a7f788a5c93a416b6. 2024-11-15T21:57:55,657 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6e3b749fcbfdadfe59f1d1cf94f5406a, disabling compactions & flushes 2024-11-15T21:57:55,657 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731707682333.6e3b749fcbfdadfe59f1d1cf94f5406a. 2024-11-15T21:57:55,657 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731707682333.6e3b749fcbfdadfe59f1d1cf94f5406a. 2024-11-15T21:57:55,658 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731707682333.6e3b749fcbfdadfe59f1d1cf94f5406a. after waiting 0 ms 2024-11-15T21:57:55,658 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731707682333.6e3b749fcbfdadfe59f1d1cf94f5406a. 2024-11-15T21:57:55,670 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a3f7be24d811f4d6177c28e08f25af2d/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-15T21:57:55,673 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731707682333.a3f7be24d811f4d6177c28e08f25af2d. 2024-11-15T21:57:55,673 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a3f7be24d811f4d6177c28e08f25af2d: Waiting for close lock at 1731707875431Running coprocessor pre-close hooks at 1731707875431Disabling compacts and flushes for region at 1731707875431Disabling writes for close at 1731707875432 (+1 ms)Obtaining lock to block concurrent updates at 1731707875432Preparing flush snapshotting stores in a3f7be24d811f4d6177c28e08f25af2d at 1731707875432Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731707682333.a3f7be24d811f4d6177c28e08f25af2d., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731707875432Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731707682333.a3f7be24d811f4d6177c28e08f25af2d. at 1731707875442 (+10 ms)Flushing a3f7be24d811f4d6177c28e08f25af2d/colfamily11: creating writer at 1731707875442Flushing a3f7be24d811f4d6177c28e08f25af2d/colfamily11: appending metadata at 1731707875473 (+31 ms)Flushing a3f7be24d811f4d6177c28e08f25af2d/colfamily11: closing flushed file at 1731707875473Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e8a0068: reopening flushed file at 1731707875607 (+134 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for a3f7be24d811f4d6177c28e08f25af2d in 190ms, sequenceid=5, compaction requested=false at 1731707875622 (+15 ms)Writing region close event to WAL at 1731707875646 (+24 ms)Running coprocessor post-close hooks at 1731707875673 (+27 ms)Closed at 1731707875673 2024-11-15T21:57:55,674 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731707682333.a3f7be24d811f4d6177c28e08f25af2d. 2024-11-15T21:57:55,674 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 229a036fd38ae13e7359b15bc2f2e587, disabling compactions & flushes 2024-11-15T21:57:55,674 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731707682333.229a036fd38ae13e7359b15bc2f2e587. 2024-11-15T21:57:55,674 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731707682333.229a036fd38ae13e7359b15bc2f2e587. 2024-11-15T21:57:55,674 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731707682333.229a036fd38ae13e7359b15bc2f2e587. after waiting 0 ms 2024-11-15T21:57:55,674 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731707682333.229a036fd38ae13e7359b15bc2f2e587. 2024-11-15T21:57:55,674 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 229a036fd38ae13e7359b15bc2f2e587 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-15T21:57:55,684 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6e3b749fcbfdadfe59f1d1cf94f5406a/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-15T21:57:55,684 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731707682333.6e3b749fcbfdadfe59f1d1cf94f5406a. 2024-11-15T21:57:55,685 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6e3b749fcbfdadfe59f1d1cf94f5406a: Waiting for close lock at 1731707875657Running coprocessor pre-close hooks at 1731707875657Disabling compacts and flushes for region at 1731707875657Disabling writes for close at 1731707875658 (+1 ms)Writing region close event to WAL at 1731707875677 (+19 ms)Running coprocessor post-close hooks at 1731707875684 (+7 ms)Closed at 1731707875684 2024-11-15T21:57:55,685 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731707682333.6e3b749fcbfdadfe59f1d1cf94f5406a. 2024-11-15T21:57:55,685 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing cbb6b5b8341f7ff2ada4ac1931a1a588, disabling compactions & flushes 2024-11-15T21:57:55,685 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731707682333.cbb6b5b8341f7ff2ada4ac1931a1a588. 2024-11-15T21:57:55,686 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731707682333.cbb6b5b8341f7ff2ada4ac1931a1a588. 2024-11-15T21:57:55,686 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731707682333.cbb6b5b8341f7ff2ada4ac1931a1a588. after waiting 0 ms 2024-11-15T21:57:55,686 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731707682333.cbb6b5b8341f7ff2ada4ac1931a1a588. 2024-11-15T21:57:55,686 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing cbb6b5b8341f7ff2ada4ac1931a1a588 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-15T21:57:55,707 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/229a036fd38ae13e7359b15bc2f2e587/.tmp/colfamily11/399bea8ff2cb457a8a966269f440064f is 52, key is aaa/colfamily11:colfamily11/1731707686509/Put/seqid=0 2024-11-15T21:57:55,722 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/cbb6b5b8341f7ff2ada4ac1931a1a588/.tmp/colfamily11/121df6bded49475db1e829de6955ffc9 is 52, key is kkk/colfamily11:colfamily11/1731707686681/Put/seqid=0 2024-11-15T21:57:55,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741892_1068 (size=39263) 2024-11-15T21:57:55,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741892_1068 (size=39263) 2024-11-15T21:57:55,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741892_1068 (size=39263) 2024-11-15T21:57:55,745 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/229a036fd38ae13e7359b15bc2f2e587/.tmp/colfamily11/399bea8ff2cb457a8a966269f440064f 2024-11-15T21:57:55,768 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/229a036fd38ae13e7359b15bc2f2e587/.tmp/colfamily11/399bea8ff2cb457a8a966269f440064f as hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/229a036fd38ae13e7359b15bc2f2e587/colfamily11/399bea8ff2cb457a8a966269f440064f 2024-11-15T21:57:55,777 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/229a036fd38ae13e7359b15bc2f2e587/colfamily11/399bea8ff2cb457a8a966269f440064f, entries=703, sequenceid=5, filesize=38.3 K 2024-11-15T21:57:55,779 DEBUG [RS:2;ae3311ea5ccb:37455 {}] regionserver.HRegionServer(1351): Waiting on 0890aeec1f0ca0649e6483a2d56f36b4, 1031049c0b4149d1189db33e38ab56cd, 2c83200b2c422973a68afce1173aae30, 46b5f83abe9585829b26884a1b26b43f, 50df10b3ae47c5dc7c4521c4c58c192f, 6694325f9c862a5a1ee02eabf7861cc3, cbb6b5b8341f7ff2ada4ac1931a1a588, d100add17d87270d000a3c39b49f8227, d9989b0341ef96818e811f874dbcf91c, dd723478fcfc0e25b606965c9135244b, f9ef867a298d175eca9d85304f2faa69, fac6af87aa0fc9826df0515f766c031e 2024-11-15T21:57:55,785 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 229a036fd38ae13e7359b15bc2f2e587 in 111ms, sequenceid=5, compaction requested=false 2024-11-15T21:57:55,786 DEBUG [RS:0;ae3311ea5ccb:40941 {}] regionserver.HRegionServer(1351): Waiting on 229a036fd38ae13e7359b15bc2f2e587, 386f4af1e738f142f3fbde37773d4d2c, 3cf264afea186a5cc96edd3e68a10d45, 3f13d1331edcf64c426c3bf9433330e4, 4256fff2fa33299a6264dd460a4003fb, 5ce283f02f0974e39c98759bb2bc9883, aec2e46cb64898ba2a66825d3732f6dc, d28ee8a2ac06d2199c72ac7f884ba470, d6063c53955b74f03cc36bd578a6bd41 2024-11-15T21:57:55,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741893_1069 (size=39263) 2024-11-15T21:57:55,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741893_1069 (size=39263) 2024-11-15T21:57:55,801 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/229a036fd38ae13e7359b15bc2f2e587/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-15T21:57:55,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741893_1069 (size=39263) 2024-11-15T21:57:55,804 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731707682333.229a036fd38ae13e7359b15bc2f2e587. 2024-11-15T21:57:55,804 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 229a036fd38ae13e7359b15bc2f2e587: Waiting for close lock at 1731707875674Running coprocessor pre-close hooks at 1731707875674Disabling compacts and flushes for region at 1731707875674Disabling writes for close at 1731707875674Obtaining lock to block concurrent updates at 1731707875674Preparing flush snapshotting stores in 229a036fd38ae13e7359b15bc2f2e587 at 1731707875674Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731707682333.229a036fd38ae13e7359b15bc2f2e587., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731707875674Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731707682333.229a036fd38ae13e7359b15bc2f2e587. at 1731707875677 (+3 ms)Flushing 229a036fd38ae13e7359b15bc2f2e587/colfamily11: creating writer at 1731707875677Flushing 229a036fd38ae13e7359b15bc2f2e587/colfamily11: appending metadata at 1731707875707 (+30 ms)Flushing 229a036fd38ae13e7359b15bc2f2e587/colfamily11: closing flushed file at 1731707875707Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3dfac47c: reopening flushed file at 1731707875760 (+53 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 229a036fd38ae13e7359b15bc2f2e587 in 111ms, sequenceid=5, compaction requested=false at 1731707875785 (+25 ms)Writing region close event to WAL at 1731707875790 (+5 ms)Running coprocessor post-close hooks at 1731707875804 (+14 ms)Closed at 1731707875804 2024-11-15T21:57:55,804 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731707682333.229a036fd38ae13e7359b15bc2f2e587. 2024-11-15T21:57:55,805 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 386f4af1e738f142f3fbde37773d4d2c, disabling compactions & flushes 2024-11-15T21:57:55,805 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731707682333.386f4af1e738f142f3fbde37773d4d2c. 2024-11-15T21:57:55,805 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731707682333.386f4af1e738f142f3fbde37773d4d2c. 2024-11-15T21:57:55,805 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731707682333.386f4af1e738f142f3fbde37773d4d2c. after waiting 0 ms 2024-11-15T21:57:55,805 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731707682333.386f4af1e738f142f3fbde37773d4d2c. 2024-11-15T21:57:55,805 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/cbb6b5b8341f7ff2ada4ac1931a1a588/.tmp/colfamily11/121df6bded49475db1e829de6955ffc9 2024-11-15T21:57:55,805 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 386f4af1e738f142f3fbde37773d4d2c 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-15T21:57:55,817 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/cbb6b5b8341f7ff2ada4ac1931a1a588/.tmp/colfamily11/121df6bded49475db1e829de6955ffc9 as hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/cbb6b5b8341f7ff2ada4ac1931a1a588/colfamily11/121df6bded49475db1e829de6955ffc9 2024-11-15T21:57:55,830 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/cbb6b5b8341f7ff2ada4ac1931a1a588/colfamily11/121df6bded49475db1e829de6955ffc9, entries=703, sequenceid=5, filesize=38.3 K 2024-11-15T21:57:55,832 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for cbb6b5b8341f7ff2ada4ac1931a1a588 in 146ms, sequenceid=5, compaction requested=false 2024-11-15T21:57:55,840 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/386f4af1e738f142f3fbde37773d4d2c/.tmp/colfamily11/bee630e764c843c2b1929d6e7a503792 is 52, key is sss/colfamily11:colfamily11/1731707686766/Put/seqid=0 2024-11-15T21:57:55,854 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/cbb6b5b8341f7ff2ada4ac1931a1a588/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-15T21:57:55,855 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731707682333.cbb6b5b8341f7ff2ada4ac1931a1a588. 2024-11-15T21:57:55,855 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for cbb6b5b8341f7ff2ada4ac1931a1a588: Waiting for close lock at 1731707875685Running coprocessor pre-close hooks at 1731707875685Disabling compacts and flushes for region at 1731707875685Disabling writes for close at 1731707875686 (+1 ms)Obtaining lock to block concurrent updates at 1731707875686Preparing flush snapshotting stores in cbb6b5b8341f7ff2ada4ac1931a1a588 at 1731707875686Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731707682333.cbb6b5b8341f7ff2ada4ac1931a1a588., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731707875686Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731707682333.cbb6b5b8341f7ff2ada4ac1931a1a588. at 1731707875690 (+4 ms)Flushing cbb6b5b8341f7ff2ada4ac1931a1a588/colfamily11: creating writer at 1731707875690Flushing cbb6b5b8341f7ff2ada4ac1931a1a588/colfamily11: appending metadata at 1731707875721 (+31 ms)Flushing cbb6b5b8341f7ff2ada4ac1931a1a588/colfamily11: closing flushed file at 1731707875721Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5551a970: reopening flushed file at 1731707875816 (+95 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for cbb6b5b8341f7ff2ada4ac1931a1a588 in 146ms, sequenceid=5, compaction requested=false at 1731707875832 (+16 ms)Writing region close event to WAL at 1731707875842 (+10 ms)Running coprocessor post-close hooks at 1731707875855 (+13 ms)Closed at 1731707875855 2024-11-15T21:57:55,856 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731707682333.cbb6b5b8341f7ff2ada4ac1931a1a588. 2024-11-15T21:57:55,856 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d100add17d87270d000a3c39b49f8227, disabling compactions & flushes 2024-11-15T21:57:55,856 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731707682333.d100add17d87270d000a3c39b49f8227. 2024-11-15T21:57:55,856 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731707682333.d100add17d87270d000a3c39b49f8227. 2024-11-15T21:57:55,856 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731707682333.d100add17d87270d000a3c39b49f8227. after waiting 0 ms 2024-11-15T21:57:55,856 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731707682333.d100add17d87270d000a3c39b49f8227. 2024-11-15T21:57:55,856 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing d100add17d87270d000a3c39b49f8227 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-15T21:57:55,892 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d100add17d87270d000a3c39b49f8227/.tmp/colfamily11/1a7511b0acdd416bbedb1730c385ee80 is 52, key is qqq/colfamily11:colfamily11/1731707686799/Put/seqid=0 2024-11-15T21:57:55,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741894_1070 (size=39263) 2024-11-15T21:57:55,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741894_1070 (size=39263) 2024-11-15T21:57:55,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741894_1070 (size=39263) 2024-11-15T21:57:55,914 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/386f4af1e738f142f3fbde37773d4d2c/.tmp/colfamily11/bee630e764c843c2b1929d6e7a503792 2024-11-15T21:57:55,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741895_1071 (size=39263) 2024-11-15T21:57:55,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741895_1071 (size=39263) 2024-11-15T21:57:55,950 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d100add17d87270d000a3c39b49f8227/.tmp/colfamily11/1a7511b0acdd416bbedb1730c385ee80 2024-11-15T21:57:55,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741895_1071 (size=39263) 2024-11-15T21:57:55,951 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/386f4af1e738f142f3fbde37773d4d2c/.tmp/colfamily11/bee630e764c843c2b1929d6e7a503792 as hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/386f4af1e738f142f3fbde37773d4d2c/colfamily11/bee630e764c843c2b1929d6e7a503792 2024-11-15T21:57:55,980 INFO [RS:2;ae3311ea5ccb:37455 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-11-15T21:57:55,980 DEBUG [RS:2;ae3311ea5ccb:37455 {}] regionserver.HRegionServer(1351): Waiting on 0890aeec1f0ca0649e6483a2d56f36b4, 1031049c0b4149d1189db33e38ab56cd, 2c83200b2c422973a68afce1173aae30, 46b5f83abe9585829b26884a1b26b43f, 50df10b3ae47c5dc7c4521c4c58c192f, 6694325f9c862a5a1ee02eabf7861cc3, d100add17d87270d000a3c39b49f8227, d9989b0341ef96818e811f874dbcf91c, dd723478fcfc0e25b606965c9135244b, f9ef867a298d175eca9d85304f2faa69, fac6af87aa0fc9826df0515f766c031e 2024-11-15T21:57:55,980 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d100add17d87270d000a3c39b49f8227/.tmp/colfamily11/1a7511b0acdd416bbedb1730c385ee80 as hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d100add17d87270d000a3c39b49f8227/colfamily11/1a7511b0acdd416bbedb1730c385ee80 2024-11-15T21:57:55,986 INFO [RS:0;ae3311ea5ccb:40941 {}] regionserver.HRegionServer(1321): Waiting on 8 regions to close 2024-11-15T21:57:55,986 DEBUG [RS:0;ae3311ea5ccb:40941 {}] regionserver.HRegionServer(1325): Online Regions={386f4af1e738f142f3fbde37773d4d2c=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731707682333.386f4af1e738f142f3fbde37773d4d2c., d28ee8a2ac06d2199c72ac7f884ba470=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731707682333.d28ee8a2ac06d2199c72ac7f884ba470., 4256fff2fa33299a6264dd460a4003fb=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731707682333.4256fff2fa33299a6264dd460a4003fb., d6063c53955b74f03cc36bd578a6bd41=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731707682333.d6063c53955b74f03cc36bd578a6bd41., aec2e46cb64898ba2a66825d3732f6dc=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731707682333.aec2e46cb64898ba2a66825d3732f6dc., 3cf264afea186a5cc96edd3e68a10d45=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731707682333.3cf264afea186a5cc96edd3e68a10d45., 5ce283f02f0974e39c98759bb2bc9883=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731707682333.5ce283f02f0974e39c98759bb2bc9883., 3f13d1331edcf64c426c3bf9433330e4=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731707682333.3f13d1331edcf64c426c3bf9433330e4.} 2024-11-15T21:57:55,987 DEBUG [RS:0;ae3311ea5ccb:40941 {}] regionserver.HRegionServer(1351): Waiting on 386f4af1e738f142f3fbde37773d4d2c, 3cf264afea186a5cc96edd3e68a10d45, 3f13d1331edcf64c426c3bf9433330e4, 4256fff2fa33299a6264dd460a4003fb, 5ce283f02f0974e39c98759bb2bc9883, aec2e46cb64898ba2a66825d3732f6dc, d28ee8a2ac06d2199c72ac7f884ba470, d6063c53955b74f03cc36bd578a6bd41 2024-11-15T21:57:55,993 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/386f4af1e738f142f3fbde37773d4d2c/colfamily11/bee630e764c843c2b1929d6e7a503792, entries=703, sequenceid=5, filesize=38.3 K 2024-11-15T21:57:55,995 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 386f4af1e738f142f3fbde37773d4d2c in 190ms, sequenceid=5, compaction requested=false 2024-11-15T21:57:56,000 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d100add17d87270d000a3c39b49f8227/colfamily11/1a7511b0acdd416bbedb1730c385ee80, entries=703, sequenceid=5, filesize=38.3 K 2024-11-15T21:57:56,002 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for d100add17d87270d000a3c39b49f8227 in 145ms, sequenceid=5, compaction requested=false 2024-11-15T21:57:56,032 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/386f4af1e738f142f3fbde37773d4d2c/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-15T21:57:56,033 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731707682333.386f4af1e738f142f3fbde37773d4d2c. 2024-11-15T21:57:56,033 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 386f4af1e738f142f3fbde37773d4d2c: Waiting for close lock at 1731707875804Running coprocessor pre-close hooks at 1731707875804Disabling compacts and flushes for region at 1731707875804Disabling writes for close at 1731707875805 (+1 ms)Obtaining lock to block concurrent updates at 1731707875805Preparing flush snapshotting stores in 386f4af1e738f142f3fbde37773d4d2c at 1731707875805Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731707682333.386f4af1e738f142f3fbde37773d4d2c., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731707875805Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731707682333.386f4af1e738f142f3fbde37773d4d2c. at 1731707875810 (+5 ms)Flushing 386f4af1e738f142f3fbde37773d4d2c/colfamily11: creating writer at 1731707875810Flushing 386f4af1e738f142f3fbde37773d4d2c/colfamily11: appending metadata at 1731707875840 (+30 ms)Flushing 386f4af1e738f142f3fbde37773d4d2c/colfamily11: closing flushed file at 1731707875840Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e4f0e10: reopening flushed file at 1731707875949 (+109 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 386f4af1e738f142f3fbde37773d4d2c in 190ms, sequenceid=5, compaction requested=false at 1731707875995 (+46 ms)Writing region close event to WAL at 1731707876005 (+10 ms)Running coprocessor post-close hooks at 1731707876033 (+28 ms)Closed at 1731707876033 2024-11-15T21:57:56,033 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d100add17d87270d000a3c39b49f8227/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-15T21:57:56,034 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731707682333.386f4af1e738f142f3fbde37773d4d2c. 2024-11-15T21:57:56,034 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d28ee8a2ac06d2199c72ac7f884ba470, disabling compactions & flushes 2024-11-15T21:57:56,034 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731707682333.d28ee8a2ac06d2199c72ac7f884ba470. 2024-11-15T21:57:56,034 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731707682333.d28ee8a2ac06d2199c72ac7f884ba470. 2024-11-15T21:57:56,034 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731707682333.d28ee8a2ac06d2199c72ac7f884ba470. after waiting 0 ms 2024-11-15T21:57:56,034 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731707682333.d28ee8a2ac06d2199c72ac7f884ba470. 2024-11-15T21:57:56,034 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing d28ee8a2ac06d2199c72ac7f884ba470 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-15T21:57:56,035 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731707682333.d100add17d87270d000a3c39b49f8227. 2024-11-15T21:57:56,035 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d100add17d87270d000a3c39b49f8227: Waiting for close lock at 1731707875856Running coprocessor pre-close hooks at 1731707875856Disabling compacts and flushes for region at 1731707875856Disabling writes for close at 1731707875856Obtaining lock to block concurrent updates at 1731707875856Preparing flush snapshotting stores in d100add17d87270d000a3c39b49f8227 at 1731707875856Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731707682333.d100add17d87270d000a3c39b49f8227., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731707875857 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731707682333.d100add17d87270d000a3c39b49f8227. at 1731707875857Flushing d100add17d87270d000a3c39b49f8227/colfamily11: creating writer at 1731707875857Flushing d100add17d87270d000a3c39b49f8227/colfamily11: appending metadata at 1731707875892 (+35 ms)Flushing d100add17d87270d000a3c39b49f8227/colfamily11: closing flushed file at 1731707875892Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d02254d: reopening flushed file at 1731707875978 (+86 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for d100add17d87270d000a3c39b49f8227 in 145ms, sequenceid=5, compaction requested=false at 1731707876002 (+24 ms)Writing region close event to WAL at 1731707876025 (+23 ms)Running coprocessor post-close hooks at 1731707876034 (+9 ms)Closed at 1731707876034 2024-11-15T21:57:56,035 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731707682333.d100add17d87270d000a3c39b49f8227. 2024-11-15T21:57:56,035 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing dd723478fcfc0e25b606965c9135244b, disabling compactions & flushes 2024-11-15T21:57:56,035 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731707682333.dd723478fcfc0e25b606965c9135244b. 2024-11-15T21:57:56,035 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731707682333.dd723478fcfc0e25b606965c9135244b. 2024-11-15T21:57:56,035 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731707682333.dd723478fcfc0e25b606965c9135244b. after waiting 0 ms 2024-11-15T21:57:56,035 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731707682333.dd723478fcfc0e25b606965c9135244b. 2024-11-15T21:57:56,050 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dd723478fcfc0e25b606965c9135244b/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-15T21:57:56,053 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731707682333.dd723478fcfc0e25b606965c9135244b. 2024-11-15T21:57:56,053 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for dd723478fcfc0e25b606965c9135244b: Waiting for close lock at 1731707876035Running coprocessor pre-close hooks at 1731707876035Disabling compacts and flushes for region at 1731707876035Disabling writes for close at 1731707876035Writing region close event to WAL at 1731707876036 (+1 ms)Running coprocessor post-close hooks at 1731707876053 (+17 ms)Closed at 1731707876053 2024-11-15T21:57:56,053 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731707682333.dd723478fcfc0e25b606965c9135244b. 2024-11-15T21:57:56,054 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f9ef867a298d175eca9d85304f2faa69, disabling compactions & flushes 2024-11-15T21:57:56,054 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731707682333.f9ef867a298d175eca9d85304f2faa69. 2024-11-15T21:57:56,054 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731707682333.f9ef867a298d175eca9d85304f2faa69. 2024-11-15T21:57:56,054 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731707682333.f9ef867a298d175eca9d85304f2faa69. after waiting 0 ms 2024-11-15T21:57:56,054 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731707682333.f9ef867a298d175eca9d85304f2faa69. 2024-11-15T21:57:56,060 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f9ef867a298d175eca9d85304f2faa69/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-15T21:57:56,061 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731707682333.f9ef867a298d175eca9d85304f2faa69. 2024-11-15T21:57:56,061 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f9ef867a298d175eca9d85304f2faa69: Waiting for close lock at 1731707876054Running coprocessor pre-close hooks at 1731707876054Disabling compacts and flushes for region at 1731707876054Disabling writes for close at 1731707876054Writing region close event to WAL at 1731707876056 (+2 ms)Running coprocessor post-close hooks at 1731707876061 (+5 ms)Closed at 1731707876061 2024-11-15T21:57:56,061 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731707682333.f9ef867a298d175eca9d85304f2faa69. 2024-11-15T21:57:56,061 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fac6af87aa0fc9826df0515f766c031e, disabling compactions & flushes 2024-11-15T21:57:56,061 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731707682333.fac6af87aa0fc9826df0515f766c031e. 2024-11-15T21:57:56,061 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731707682333.fac6af87aa0fc9826df0515f766c031e. 2024-11-15T21:57:56,061 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731707682333.fac6af87aa0fc9826df0515f766c031e. after waiting 0 ms 2024-11-15T21:57:56,061 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731707682333.fac6af87aa0fc9826df0515f766c031e. 2024-11-15T21:57:56,061 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing fac6af87aa0fc9826df0515f766c031e 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-15T21:57:56,074 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d28ee8a2ac06d2199c72ac7f884ba470/.tmp/colfamily11/b1d9e354995a41d2a6654c58ed5483e8 is 52, key is iii/colfamily11:colfamily11/1731707686640/Put/seqid=0 2024-11-15T21:57:56,094 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fac6af87aa0fc9826df0515f766c031e/.tmp/colfamily11/e759039510194928a22d8b77b9c22af7 is 52, key is uuu/colfamily11:colfamily11/1731707686853/Put/seqid=0 2024-11-15T21:57:56,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741896_1072 (size=39263) 2024-11-15T21:57:56,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741896_1072 (size=39263) 2024-11-15T21:57:56,101 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d28ee8a2ac06d2199c72ac7f884ba470/.tmp/colfamily11/b1d9e354995a41d2a6654c58ed5483e8 2024-11-15T21:57:56,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741896_1072 (size=39263) 2024-11-15T21:57:56,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741897_1073 (size=39263) 2024-11-15T21:57:56,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741897_1073 (size=39263) 2024-11-15T21:57:56,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741897_1073 (size=39263) 2024-11-15T21:57:56,111 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d28ee8a2ac06d2199c72ac7f884ba470/.tmp/colfamily11/b1d9e354995a41d2a6654c58ed5483e8 as hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d28ee8a2ac06d2199c72ac7f884ba470/colfamily11/b1d9e354995a41d2a6654c58ed5483e8 2024-11-15T21:57:56,126 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fac6af87aa0fc9826df0515f766c031e/.tmp/colfamily11/e759039510194928a22d8b77b9c22af7 2024-11-15T21:57:56,131 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d28ee8a2ac06d2199c72ac7f884ba470/colfamily11/b1d9e354995a41d2a6654c58ed5483e8, entries=703, sequenceid=5, filesize=38.3 K 2024-11-15T21:57:56,134 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fac6af87aa0fc9826df0515f766c031e/.tmp/colfamily11/e759039510194928a22d8b77b9c22af7 as hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fac6af87aa0fc9826df0515f766c031e/colfamily11/e759039510194928a22d8b77b9c22af7 2024-11-15T21:57:56,137 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for d28ee8a2ac06d2199c72ac7f884ba470 in 103ms, sequenceid=5, compaction requested=false 2024-11-15T21:57:56,141 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fac6af87aa0fc9826df0515f766c031e/colfamily11/e759039510194928a22d8b77b9c22af7, entries=703, sequenceid=5, filesize=38.3 K 2024-11-15T21:57:56,143 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for fac6af87aa0fc9826df0515f766c031e in 81ms, sequenceid=5, compaction requested=false 2024-11-15T21:57:56,178 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d28ee8a2ac06d2199c72ac7f884ba470/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-15T21:57:56,180 DEBUG [RS:2;ae3311ea5ccb:37455 {}] regionserver.HRegionServer(1351): Waiting on 0890aeec1f0ca0649e6483a2d56f36b4, 1031049c0b4149d1189db33e38ab56cd, 2c83200b2c422973a68afce1173aae30, 46b5f83abe9585829b26884a1b26b43f, 50df10b3ae47c5dc7c4521c4c58c192f, 6694325f9c862a5a1ee02eabf7861cc3, d9989b0341ef96818e811f874dbcf91c, fac6af87aa0fc9826df0515f766c031e 2024-11-15T21:57:56,186 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731707682333.d28ee8a2ac06d2199c72ac7f884ba470. 2024-11-15T21:57:56,186 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d28ee8a2ac06d2199c72ac7f884ba470: Waiting for close lock at 1731707876034Running coprocessor pre-close hooks at 1731707876034Disabling compacts and flushes for region at 1731707876034Disabling writes for close at 1731707876034Obtaining lock to block concurrent updates at 1731707876034Preparing flush snapshotting stores in d28ee8a2ac06d2199c72ac7f884ba470 at 1731707876034Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731707682333.d28ee8a2ac06d2199c72ac7f884ba470., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731707876035 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731707682333.d28ee8a2ac06d2199c72ac7f884ba470. at 1731707876045 (+10 ms)Flushing d28ee8a2ac06d2199c72ac7f884ba470/colfamily11: creating writer at 1731707876045Flushing d28ee8a2ac06d2199c72ac7f884ba470/colfamily11: appending metadata at 1731707876073 (+28 ms)Flushing d28ee8a2ac06d2199c72ac7f884ba470/colfamily11: closing flushed file at 1731707876073Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3435a6dd: reopening flushed file at 1731707876110 (+37 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for d28ee8a2ac06d2199c72ac7f884ba470 in 103ms, sequenceid=5, compaction requested=false at 1731707876137 (+27 ms)Writing region close event to WAL at 1731707876144 (+7 ms)Running coprocessor post-close hooks at 1731707876186 (+42 ms)Closed at 1731707876186 2024-11-15T21:57:56,187 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731707682333.d28ee8a2ac06d2199c72ac7f884ba470. 2024-11-15T21:57:56,187 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4256fff2fa33299a6264dd460a4003fb, disabling compactions & flushes 2024-11-15T21:57:56,187 DEBUG [RS:0;ae3311ea5ccb:40941 {}] regionserver.HRegionServer(1351): Waiting on 3cf264afea186a5cc96edd3e68a10d45, 3f13d1331edcf64c426c3bf9433330e4, 4256fff2fa33299a6264dd460a4003fb, 5ce283f02f0974e39c98759bb2bc9883, aec2e46cb64898ba2a66825d3732f6dc, d6063c53955b74f03cc36bd578a6bd41 2024-11-15T21:57:56,187 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731707682333.4256fff2fa33299a6264dd460a4003fb. 2024-11-15T21:57:56,187 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731707682333.4256fff2fa33299a6264dd460a4003fb. 2024-11-15T21:57:56,187 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731707682333.4256fff2fa33299a6264dd460a4003fb. after waiting 0 ms 2024-11-15T21:57:56,187 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731707682333.4256fff2fa33299a6264dd460a4003fb. 2024-11-15T21:57:56,196 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fac6af87aa0fc9826df0515f766c031e/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-15T21:57:56,198 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731707682333.fac6af87aa0fc9826df0515f766c031e. 2024-11-15T21:57:56,198 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fac6af87aa0fc9826df0515f766c031e: Waiting for close lock at 1731707876061Running coprocessor pre-close hooks at 1731707876061Disabling compacts and flushes for region at 1731707876061Disabling writes for close at 1731707876061Obtaining lock to block concurrent updates at 1731707876061Preparing flush snapshotting stores in fac6af87aa0fc9826df0515f766c031e at 1731707876061Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731707682333.fac6af87aa0fc9826df0515f766c031e., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731707876062 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731707682333.fac6af87aa0fc9826df0515f766c031e. at 1731707876062Flushing fac6af87aa0fc9826df0515f766c031e/colfamily11: creating writer at 1731707876062Flushing fac6af87aa0fc9826df0515f766c031e/colfamily11: appending metadata at 1731707876093 (+31 ms)Flushing fac6af87aa0fc9826df0515f766c031e/colfamily11: closing flushed file at 1731707876093Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@58bec46d: reopening flushed file at 1731707876133 (+40 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for fac6af87aa0fc9826df0515f766c031e in 81ms, sequenceid=5, compaction requested=false at 1731707876143 (+10 ms)Writing region close event to WAL at 1731707876161 (+18 ms)Running coprocessor post-close hooks at 1731707876198 (+37 ms)Closed at 1731707876198 2024-11-15T21:57:56,198 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731707682333.fac6af87aa0fc9826df0515f766c031e. 2024-11-15T21:57:56,198 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2c83200b2c422973a68afce1173aae30, disabling compactions & flushes 2024-11-15T21:57:56,198 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731707682333.2c83200b2c422973a68afce1173aae30. 2024-11-15T21:57:56,198 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731707682333.2c83200b2c422973a68afce1173aae30. 2024-11-15T21:57:56,198 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731707682333.2c83200b2c422973a68afce1173aae30. after waiting 0 ms 2024-11-15T21:57:56,199 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731707682333.2c83200b2c422973a68afce1173aae30. 2024-11-15T21:57:56,199 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 2c83200b2c422973a68afce1173aae30 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-15T21:57:56,225 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4256fff2fa33299a6264dd460a4003fb/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-15T21:57:56,226 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731707682333.4256fff2fa33299a6264dd460a4003fb. 2024-11-15T21:57:56,226 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4256fff2fa33299a6264dd460a4003fb: Waiting for close lock at 1731707876187Running coprocessor pre-close hooks at 1731707876187Disabling compacts and flushes for region at 1731707876187Disabling writes for close at 1731707876187Writing region close event to WAL at 1731707876200 (+13 ms)Running coprocessor post-close hooks at 1731707876226 (+26 ms)Closed at 1731707876226 2024-11-15T21:57:56,226 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731707682333.4256fff2fa33299a6264dd460a4003fb. 2024-11-15T21:57:56,227 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d6063c53955b74f03cc36bd578a6bd41, disabling compactions & flushes 2024-11-15T21:57:56,227 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731707682333.d6063c53955b74f03cc36bd578a6bd41. 2024-11-15T21:57:56,227 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731707682333.d6063c53955b74f03cc36bd578a6bd41. 2024-11-15T21:57:56,227 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731707682333.d6063c53955b74f03cc36bd578a6bd41. after waiting 0 ms 2024-11-15T21:57:56,227 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731707682333.d6063c53955b74f03cc36bd578a6bd41. 2024-11-15T21:57:56,261 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2c83200b2c422973a68afce1173aae30/.tmp/colfamily11/2ebcd0d75a7e49c79e3f37735a47f1a3 is 52, key is eee/colfamily11:colfamily11/1731707686555/Put/seqid=0 2024-11-15T21:57:56,273 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d6063c53955b74f03cc36bd578a6bd41/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-15T21:57:56,281 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731707682333.d6063c53955b74f03cc36bd578a6bd41. 2024-11-15T21:57:56,281 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d6063c53955b74f03cc36bd578a6bd41: Waiting for close lock at 1731707876227Running coprocessor pre-close hooks at 1731707876227Disabling compacts and flushes for region at 1731707876227Disabling writes for close at 1731707876227Writing region close event to WAL at 1731707876253 (+26 ms)Running coprocessor post-close hooks at 1731707876281 (+28 ms)Closed at 1731707876281 2024-11-15T21:57:56,281 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731707682333.d6063c53955b74f03cc36bd578a6bd41. 2024-11-15T21:57:56,281 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing aec2e46cb64898ba2a66825d3732f6dc, disabling compactions & flushes 2024-11-15T21:57:56,282 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731707682333.aec2e46cb64898ba2a66825d3732f6dc. 2024-11-15T21:57:56,282 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731707682333.aec2e46cb64898ba2a66825d3732f6dc. 2024-11-15T21:57:56,282 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731707682333.aec2e46cb64898ba2a66825d3732f6dc. after waiting 0 ms 2024-11-15T21:57:56,282 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731707682333.aec2e46cb64898ba2a66825d3732f6dc. 2024-11-15T21:57:56,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741898_1074 (size=39263) 2024-11-15T21:57:56,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741898_1074 (size=39263) 2024-11-15T21:57:56,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741898_1074 (size=39263) 2024-11-15T21:57:56,311 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2c83200b2c422973a68afce1173aae30/.tmp/colfamily11/2ebcd0d75a7e49c79e3f37735a47f1a3 2024-11-15T21:57:56,312 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aec2e46cb64898ba2a66825d3732f6dc/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-15T21:57:56,315 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731707682333.aec2e46cb64898ba2a66825d3732f6dc. 2024-11-15T21:57:56,315 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for aec2e46cb64898ba2a66825d3732f6dc: Waiting for close lock at 1731707876281Running coprocessor pre-close hooks at 1731707876281Disabling compacts and flushes for region at 1731707876281Disabling writes for close at 1731707876282 (+1 ms)Writing region close event to WAL at 1731707876297 (+15 ms)Running coprocessor post-close hooks at 1731707876315 (+18 ms)Closed at 1731707876315 2024-11-15T21:57:56,316 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731707682333.aec2e46cb64898ba2a66825d3732f6dc. 2024-11-15T21:57:56,316 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3cf264afea186a5cc96edd3e68a10d45, disabling compactions & flushes 2024-11-15T21:57:56,316 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731707682333.3cf264afea186a5cc96edd3e68a10d45. 2024-11-15T21:57:56,316 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731707682333.3cf264afea186a5cc96edd3e68a10d45. 2024-11-15T21:57:56,316 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731707682333.3cf264afea186a5cc96edd3e68a10d45. after waiting 0 ms 2024-11-15T21:57:56,316 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731707682333.3cf264afea186a5cc96edd3e68a10d45. 2024-11-15T21:57:56,334 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3cf264afea186a5cc96edd3e68a10d45/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-15T21:57:56,339 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731707682333.3cf264afea186a5cc96edd3e68a10d45. 2024-11-15T21:57:56,339 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3cf264afea186a5cc96edd3e68a10d45: Waiting for close lock at 1731707876316Running coprocessor pre-close hooks at 1731707876316Disabling compacts and flushes for region at 1731707876316Disabling writes for close at 1731707876316Writing region close event to WAL at 1731707876325 (+9 ms)Running coprocessor post-close hooks at 1731707876339 (+14 ms)Closed at 1731707876339 2024-11-15T21:57:56,339 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731707682333.3cf264afea186a5cc96edd3e68a10d45. 2024-11-15T21:57:56,339 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5ce283f02f0974e39c98759bb2bc9883, disabling compactions & flushes 2024-11-15T21:57:56,339 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731707682333.5ce283f02f0974e39c98759bb2bc9883. 2024-11-15T21:57:56,340 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731707682333.5ce283f02f0974e39c98759bb2bc9883. 2024-11-15T21:57:56,340 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731707682333.5ce283f02f0974e39c98759bb2bc9883. after waiting 0 ms 2024-11-15T21:57:56,340 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731707682333.5ce283f02f0974e39c98759bb2bc9883. 2024-11-15T21:57:56,340 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 5ce283f02f0974e39c98759bb2bc9883 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-15T21:57:56,347 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2c83200b2c422973a68afce1173aae30/.tmp/colfamily11/2ebcd0d75a7e49c79e3f37735a47f1a3 as hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2c83200b2c422973a68afce1173aae30/colfamily11/2ebcd0d75a7e49c79e3f37735a47f1a3 2024-11-15T21:57:56,376 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ce283f02f0974e39c98759bb2bc9883/.tmp/colfamily11/b92d53627b2749d3b7fa080f2aeb0ca7 is 52, key is vvv/colfamily11:colfamily11/1731707686815/Put/seqid=0 2024-11-15T21:57:56,381 DEBUG [RS:2;ae3311ea5ccb:37455 {}] regionserver.HRegionServer(1351): Waiting on 0890aeec1f0ca0649e6483a2d56f36b4, 1031049c0b4149d1189db33e38ab56cd, 2c83200b2c422973a68afce1173aae30, 46b5f83abe9585829b26884a1b26b43f, 50df10b3ae47c5dc7c4521c4c58c192f, 6694325f9c862a5a1ee02eabf7861cc3, d9989b0341ef96818e811f874dbcf91c 2024-11-15T21:57:56,387 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2c83200b2c422973a68afce1173aae30/colfamily11/2ebcd0d75a7e49c79e3f37735a47f1a3, entries=703, sequenceid=5, filesize=38.3 K 2024-11-15T21:57:56,389 DEBUG [RS:0;ae3311ea5ccb:40941 {}] regionserver.HRegionServer(1351): Waiting on 3f13d1331edcf64c426c3bf9433330e4, 5ce283f02f0974e39c98759bb2bc9883 2024-11-15T21:57:56,393 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 2c83200b2c422973a68afce1173aae30 in 194ms, sequenceid=5, compaction requested=false 2024-11-15T21:57:56,444 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2c83200b2c422973a68afce1173aae30/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-15T21:57:56,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741899_1075 (size=39263) 2024-11-15T21:57:56,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741899_1075 (size=39263) 2024-11-15T21:57:56,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741899_1075 (size=39263) 2024-11-15T21:57:56,446 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731707682333.2c83200b2c422973a68afce1173aae30. 2024-11-15T21:57:56,446 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2c83200b2c422973a68afce1173aae30: Waiting for close lock at 1731707876198Running coprocessor pre-close hooks at 1731707876198Disabling compacts and flushes for region at 1731707876198Disabling writes for close at 1731707876199 (+1 ms)Obtaining lock to block concurrent updates at 1731707876199Preparing flush snapshotting stores in 2c83200b2c422973a68afce1173aae30 at 1731707876199Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731707682333.2c83200b2c422973a68afce1173aae30., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731707876202 (+3 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731707682333.2c83200b2c422973a68afce1173aae30. at 1731707876203 (+1 ms)Flushing 2c83200b2c422973a68afce1173aae30/colfamily11: creating writer at 1731707876203Flushing 2c83200b2c422973a68afce1173aae30/colfamily11: appending metadata at 1731707876258 (+55 ms)Flushing 2c83200b2c422973a68afce1173aae30/colfamily11: closing flushed file at 1731707876258Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3fb28e14: reopening flushed file at 1731707876340 (+82 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 2c83200b2c422973a68afce1173aae30 in 194ms, sequenceid=5, compaction requested=false at 1731707876394 (+54 ms)Writing region close event to WAL at 1731707876423 (+29 ms)Running coprocessor post-close hooks at 1731707876446 (+23 ms)Closed at 1731707876446 2024-11-15T21:57:56,447 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731707682333.2c83200b2c422973a68afce1173aae30. 2024-11-15T21:57:56,447 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6694325f9c862a5a1ee02eabf7861cc3, disabling compactions & flushes 2024-11-15T21:57:56,447 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731707682333.6694325f9c862a5a1ee02eabf7861cc3. 2024-11-15T21:57:56,447 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731707682333.6694325f9c862a5a1ee02eabf7861cc3. 2024-11-15T21:57:56,447 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731707682333.6694325f9c862a5a1ee02eabf7861cc3. after waiting 0 ms 2024-11-15T21:57:56,447 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731707682333.6694325f9c862a5a1ee02eabf7861cc3. 2024-11-15T21:57:56,482 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6694325f9c862a5a1ee02eabf7861cc3/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-15T21:57:56,483 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731707682333.6694325f9c862a5a1ee02eabf7861cc3. 2024-11-15T21:57:56,483 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6694325f9c862a5a1ee02eabf7861cc3: Waiting for close lock at 1731707876447Running coprocessor pre-close hooks at 1731707876447Disabling compacts and flushes for region at 1731707876447Disabling writes for close at 1731707876447Writing region close event to WAL at 1731707876462 (+15 ms)Running coprocessor post-close hooks at 1731707876483 (+21 ms)Closed at 1731707876483 2024-11-15T21:57:56,484 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731707682333.6694325f9c862a5a1ee02eabf7861cc3. 2024-11-15T21:57:56,484 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d9989b0341ef96818e811f874dbcf91c, disabling compactions & flushes 2024-11-15T21:57:56,484 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731707682333.d9989b0341ef96818e811f874dbcf91c. 2024-11-15T21:57:56,484 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731707682333.d9989b0341ef96818e811f874dbcf91c. 2024-11-15T21:57:56,484 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731707682333.d9989b0341ef96818e811f874dbcf91c. after waiting 0 ms 2024-11-15T21:57:56,484 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731707682333.d9989b0341ef96818e811f874dbcf91c. 2024-11-15T21:57:56,514 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d9989b0341ef96818e811f874dbcf91c/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-15T21:57:56,518 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731707682333.d9989b0341ef96818e811f874dbcf91c. 2024-11-15T21:57:56,518 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d9989b0341ef96818e811f874dbcf91c: Waiting for close lock at 1731707876484Running coprocessor pre-close hooks at 1731707876484Disabling compacts and flushes for region at 1731707876484Disabling writes for close at 1731707876484Writing region close event to WAL at 1731707876489 (+5 ms)Running coprocessor post-close hooks at 1731707876518 (+29 ms)Closed at 1731707876518 2024-11-15T21:57:56,518 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731707682333.d9989b0341ef96818e811f874dbcf91c. 2024-11-15T21:57:56,518 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1031049c0b4149d1189db33e38ab56cd, disabling compactions & flushes 2024-11-15T21:57:56,518 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731707682333.1031049c0b4149d1189db33e38ab56cd. 2024-11-15T21:57:56,518 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731707682333.1031049c0b4149d1189db33e38ab56cd. 2024-11-15T21:57:56,518 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731707682333.1031049c0b4149d1189db33e38ab56cd. after waiting 0 ms 2024-11-15T21:57:56,518 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731707682333.1031049c0b4149d1189db33e38ab56cd. 2024-11-15T21:57:56,519 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 1031049c0b4149d1189db33e38ab56cd 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-15T21:57:56,551 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1031049c0b4149d1189db33e38ab56cd/.tmp/colfamily11/7da9e4f6931945f28a708a317bcab876 is 52, key is yyy/colfamily11:colfamily11/1731707687012/Put/seqid=0 2024-11-15T21:57:56,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741900_1076 (size=39263) 2024-11-15T21:57:56,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741900_1076 (size=39263) 2024-11-15T21:57:56,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741900_1076 (size=39263) 2024-11-15T21:57:56,585 DEBUG [RS:2;ae3311ea5ccb:37455 {}] regionserver.HRegionServer(1351): Waiting on 0890aeec1f0ca0649e6483a2d56f36b4, 1031049c0b4149d1189db33e38ab56cd, 46b5f83abe9585829b26884a1b26b43f, 50df10b3ae47c5dc7c4521c4c58c192f 2024-11-15T21:57:56,589 DEBUG [RS:0;ae3311ea5ccb:40941 {}] regionserver.HRegionServer(1351): Waiting on 3f13d1331edcf64c426c3bf9433330e4, 5ce283f02f0974e39c98759bb2bc9883 2024-11-15T21:57:56,785 DEBUG [RS:2;ae3311ea5ccb:37455 {}] regionserver.HRegionServer(1351): Waiting on 0890aeec1f0ca0649e6483a2d56f36b4, 1031049c0b4149d1189db33e38ab56cd, 46b5f83abe9585829b26884a1b26b43f, 50df10b3ae47c5dc7c4521c4c58c192f 2024-11-15T21:57:56,789 DEBUG [RS:0;ae3311ea5ccb:40941 {}] regionserver.HRegionServer(1351): Waiting on 3f13d1331edcf64c426c3bf9433330e4, 5ce283f02f0974e39c98759bb2bc9883 2024-11-15T21:57:56,846 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ce283f02f0974e39c98759bb2bc9883/.tmp/colfamily11/b92d53627b2749d3b7fa080f2aeb0ca7 2024-11-15T21:57:56,854 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ce283f02f0974e39c98759bb2bc9883/.tmp/colfamily11/b92d53627b2749d3b7fa080f2aeb0ca7 as hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ce283f02f0974e39c98759bb2bc9883/colfamily11/b92d53627b2749d3b7fa080f2aeb0ca7 2024-11-15T21:57:56,861 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ce283f02f0974e39c98759bb2bc9883/colfamily11/b92d53627b2749d3b7fa080f2aeb0ca7, entries=703, sequenceid=5, filesize=38.3 K 2024-11-15T21:57:56,862 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 5ce283f02f0974e39c98759bb2bc9883 in 522ms, sequenceid=5, compaction requested=false 2024-11-15T21:57:56,869 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ce283f02f0974e39c98759bb2bc9883/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-15T21:57:56,870 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731707682333.5ce283f02f0974e39c98759bb2bc9883. 2024-11-15T21:57:56,870 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5ce283f02f0974e39c98759bb2bc9883: Waiting for close lock at 1731707876339Running coprocessor pre-close hooks at 1731707876339Disabling compacts and flushes for region at 1731707876339Disabling writes for close at 1731707876340 (+1 ms)Obtaining lock to block concurrent updates at 1731707876340Preparing flush snapshotting stores in 5ce283f02f0974e39c98759bb2bc9883 at 1731707876340Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731707682333.5ce283f02f0974e39c98759bb2bc9883., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731707876341 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731707682333.5ce283f02f0974e39c98759bb2bc9883. at 1731707876343 (+2 ms)Flushing 5ce283f02f0974e39c98759bb2bc9883/colfamily11: creating writer at 1731707876343Flushing 5ce283f02f0974e39c98759bb2bc9883/colfamily11: appending metadata at 1731707876374 (+31 ms)Flushing 5ce283f02f0974e39c98759bb2bc9883/colfamily11: closing flushed file at 1731707876374Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@744b8f1b: reopening flushed file at 1731707876853 (+479 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 5ce283f02f0974e39c98759bb2bc9883 in 522ms, sequenceid=5, compaction requested=false at 1731707876862 (+9 ms)Writing region close event to WAL at 1731707876864 (+2 ms)Running coprocessor post-close hooks at 1731707876870 (+6 ms)Closed at 1731707876870 2024-11-15T21:57:56,870 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731707682333.5ce283f02f0974e39c98759bb2bc9883. 2024-11-15T21:57:56,870 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3f13d1331edcf64c426c3bf9433330e4, disabling compactions & flushes 2024-11-15T21:57:56,870 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731707682333.3f13d1331edcf64c426c3bf9433330e4. 2024-11-15T21:57:56,870 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731707682333.3f13d1331edcf64c426c3bf9433330e4. 2024-11-15T21:57:56,870 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731707682333.3f13d1331edcf64c426c3bf9433330e4. after waiting 0 ms 2024-11-15T21:57:56,870 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731707682333.3f13d1331edcf64c426c3bf9433330e4. 2024-11-15T21:57:56,871 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 3f13d1331edcf64c426c3bf9433330e4 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-15T21:57:56,894 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3f13d1331edcf64c426c3bf9433330e4/.tmp/colfamily11/fc1a9cda97484b61aca47bd75fa9cea6 is 52, key is fff/colfamily11:colfamily11/1731707686569/Put/seqid=0 2024-11-15T21:57:56,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741901_1077 (size=39263) 2024-11-15T21:57:56,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741901_1077 (size=39263) 2024-11-15T21:57:56,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741901_1077 (size=39263) 2024-11-15T21:57:56,921 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3f13d1331edcf64c426c3bf9433330e4/.tmp/colfamily11/fc1a9cda97484b61aca47bd75fa9cea6 2024-11-15T21:57:56,929 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3f13d1331edcf64c426c3bf9433330e4/.tmp/colfamily11/fc1a9cda97484b61aca47bd75fa9cea6 as hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3f13d1331edcf64c426c3bf9433330e4/colfamily11/fc1a9cda97484b61aca47bd75fa9cea6 2024-11-15T21:57:56,939 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3f13d1331edcf64c426c3bf9433330e4/colfamily11/fc1a9cda97484b61aca47bd75fa9cea6, entries=703, sequenceid=5, filesize=38.3 K 2024-11-15T21:57:56,940 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 3f13d1331edcf64c426c3bf9433330e4 in 70ms, sequenceid=5, compaction requested=false 2024-11-15T21:57:56,952 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3f13d1331edcf64c426c3bf9433330e4/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-15T21:57:56,953 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731707682333.3f13d1331edcf64c426c3bf9433330e4. 2024-11-15T21:57:56,953 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3f13d1331edcf64c426c3bf9433330e4: Waiting for close lock at 1731707876870Running coprocessor pre-close hooks at 1731707876870Disabling compacts and flushes for region at 1731707876870Disabling writes for close at 1731707876870Obtaining lock to block concurrent updates at 1731707876871 (+1 ms)Preparing flush snapshotting stores in 3f13d1331edcf64c426c3bf9433330e4 at 1731707876871Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731707682333.3f13d1331edcf64c426c3bf9433330e4., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731707876871Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731707682333.3f13d1331edcf64c426c3bf9433330e4. at 1731707876873 (+2 ms)Flushing 3f13d1331edcf64c426c3bf9433330e4/colfamily11: creating writer at 1731707876873Flushing 3f13d1331edcf64c426c3bf9433330e4/colfamily11: appending metadata at 1731707876893 (+20 ms)Flushing 3f13d1331edcf64c426c3bf9433330e4/colfamily11: closing flushed file at 1731707876894 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@17ef11cd: reopening flushed file at 1731707876928 (+34 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 3f13d1331edcf64c426c3bf9433330e4 in 70ms, sequenceid=5, compaction requested=false at 1731707876941 (+13 ms)Writing region close event to WAL at 1731707876947 (+6 ms)Running coprocessor post-close hooks at 1731707876953 (+6 ms)Closed at 1731707876953 2024-11-15T21:57:56,954 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731707682333.3f13d1331edcf64c426c3bf9433330e4. 2024-11-15T21:57:56,981 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1031049c0b4149d1189db33e38ab56cd/.tmp/colfamily11/7da9e4f6931945f28a708a317bcab876 2024-11-15T21:57:56,986 INFO [RS:2;ae3311ea5ccb:37455 {}] regionserver.HRegionServer(1321): Waiting on 4 regions to close 2024-11-15T21:57:56,986 DEBUG [RS:2;ae3311ea5ccb:37455 {}] regionserver.HRegionServer(1325): Online Regions={1031049c0b4149d1189db33e38ab56cd=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731707682333.1031049c0b4149d1189db33e38ab56cd., 46b5f83abe9585829b26884a1b26b43f=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731707682333.46b5f83abe9585829b26884a1b26b43f., 50df10b3ae47c5dc7c4521c4c58c192f=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731707682333.50df10b3ae47c5dc7c4521c4c58c192f., 0890aeec1f0ca0649e6483a2d56f36b4=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731707682333.0890aeec1f0ca0649e6483a2d56f36b4.} 2024-11-15T21:57:56,986 DEBUG [RS:2;ae3311ea5ccb:37455 {}] regionserver.HRegionServer(1351): Waiting on 0890aeec1f0ca0649e6483a2d56f36b4, 1031049c0b4149d1189db33e38ab56cd, 46b5f83abe9585829b26884a1b26b43f, 50df10b3ae47c5dc7c4521c4c58c192f 2024-11-15T21:57:56,989 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1031049c0b4149d1189db33e38ab56cd/.tmp/colfamily11/7da9e4f6931945f28a708a317bcab876 as hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1031049c0b4149d1189db33e38ab56cd/colfamily11/7da9e4f6931945f28a708a317bcab876 2024-11-15T21:57:56,990 INFO [RS:0;ae3311ea5ccb:40941 {}] regionserver.HRegionServer(976): stopping server ae3311ea5ccb,40941,1731707679516; all regions closed. 2024-11-15T21:57:56,999 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1031049c0b4149d1189db33e38ab56cd/colfamily11/7da9e4f6931945f28a708a317bcab876, entries=703, sequenceid=5, filesize=38.3 K 2024-11-15T21:57:57,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741865_1041 (size=15752) 2024-11-15T21:57:57,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741865_1041 (size=15752) 2024-11-15T21:57:57,006 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 1031049c0b4149d1189db33e38ab56cd in 488ms, sequenceid=5, compaction requested=false 2024-11-15T21:57:57,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741865_1041 (size=15752) 2024-11-15T21:57:57,011 DEBUG [RS:0;ae3311ea5ccb:40941 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/oldWALs 2024-11-15T21:57:57,011 INFO [RS:0;ae3311ea5ccb:40941 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ae3311ea5ccb%2C40941%2C1731707679516.meta:.meta(num 1731707868328) 2024-11-15T21:57:57,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741834_1010 (size=277788) 2024-11-15T21:57:57,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741834_1010 (size=277788) 2024-11-15T21:57:57,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741834_1010 (size=277788) 2024-11-15T21:57:57,020 DEBUG [RS:0;ae3311ea5ccb:40941 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/oldWALs 2024-11-15T21:57:57,021 INFO [RS:0;ae3311ea5ccb:40941 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ae3311ea5ccb%2C40941%2C1731707679516:(num 1731707681383) 2024-11-15T21:57:57,021 DEBUG [RS:0;ae3311ea5ccb:40941 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T21:57:57,021 INFO [RS:0;ae3311ea5ccb:40941 {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T21:57:57,021 INFO [RS:0;ae3311ea5ccb:40941 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-15T21:57:57,021 INFO [RS:0;ae3311ea5ccb:40941 {}] hbase.ChoreService(370): Chore service for: regionserver/ae3311ea5ccb:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-15T21:57:57,021 INFO [RS:0;ae3311ea5ccb:40941 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-15T21:57:57,021 INFO [regionserver/ae3311ea5ccb:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-15T21:57:57,021 INFO [RS:0;ae3311ea5ccb:40941 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:40941 2024-11-15T21:57:57,022 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1031049c0b4149d1189db33e38ab56cd/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-15T21:57:57,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36531-0x100f86521fe0000, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T21:57:57,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40941-0x100f86521fe0001, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ae3311ea5ccb,40941,1731707679516 2024-11-15T21:57:57,033 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ae3311ea5ccb,40941,1731707679516] 2024-11-15T21:57:57,033 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731707682333.1031049c0b4149d1189db33e38ab56cd. 2024-11-15T21:57:57,033 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1031049c0b4149d1189db33e38ab56cd: Waiting for close lock at 1731707876518Running coprocessor pre-close hooks at 1731707876518Disabling compacts and flushes for region at 1731707876518Disabling writes for close at 1731707876518Obtaining lock to block concurrent updates at 1731707876519 (+1 ms)Preparing flush snapshotting stores in 1031049c0b4149d1189db33e38ab56cd at 1731707876519Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731707682333.1031049c0b4149d1189db33e38ab56cd., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731707876519Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731707682333.1031049c0b4149d1189db33e38ab56cd. at 1731707876522 (+3 ms)Flushing 1031049c0b4149d1189db33e38ab56cd/colfamily11: creating writer at 1731707876522Flushing 1031049c0b4149d1189db33e38ab56cd/colfamily11: appending metadata at 1731707876551 (+29 ms)Flushing 1031049c0b4149d1189db33e38ab56cd/colfamily11: closing flushed file at 1731707876551Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e7788a9: reopening flushed file at 1731707876988 (+437 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 1031049c0b4149d1189db33e38ab56cd in 488ms, sequenceid=5, compaction requested=false at 1731707877006 (+18 ms)Writing region close event to WAL at 1731707877014 (+8 ms)Running coprocessor post-close hooks at 1731707877033 (+19 ms)Closed at 1731707877033 2024-11-15T21:57:57,033 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731707682333.1031049c0b4149d1189db33e38ab56cd. 2024-11-15T21:57:57,034 INFO [RS:0;ae3311ea5ccb:40941 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-15T21:57:57,034 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 46b5f83abe9585829b26884a1b26b43f, disabling compactions & flushes 2024-11-15T21:57:57,034 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731707682333.46b5f83abe9585829b26884a1b26b43f. 2024-11-15T21:57:57,034 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731707682333.46b5f83abe9585829b26884a1b26b43f. 2024-11-15T21:57:57,034 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731707682333.46b5f83abe9585829b26884a1b26b43f. after waiting 0 ms 2024-11-15T21:57:57,034 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731707682333.46b5f83abe9585829b26884a1b26b43f. 2024-11-15T21:57:57,034 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 46b5f83abe9585829b26884a1b26b43f 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-15T21:57:57,036 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ae3311ea5ccb,40941,1731707679516 already deleted, retry=false 2024-11-15T21:57:57,036 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ae3311ea5ccb,40941,1731707679516 expired; onlineServers=1 2024-11-15T21:57:57,068 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/46b5f83abe9585829b26884a1b26b43f/.tmp/colfamily11/8fdc31e096a4428caf7a01bc018cbea5 is 52, key is xxx/colfamily11:colfamily11/1731707686968/Put/seqid=0 2024-11-15T21:57:57,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741902_1078 (size=39263) 2024-11-15T21:57:57,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741902_1078 (size=39263) 2024-11-15T21:57:57,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741902_1078 (size=39263) 2024-11-15T21:57:57,108 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/46b5f83abe9585829b26884a1b26b43f/.tmp/colfamily11/8fdc31e096a4428caf7a01bc018cbea5 2024-11-15T21:57:57,116 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/46b5f83abe9585829b26884a1b26b43f/.tmp/colfamily11/8fdc31e096a4428caf7a01bc018cbea5 as hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/46b5f83abe9585829b26884a1b26b43f/colfamily11/8fdc31e096a4428caf7a01bc018cbea5 2024-11-15T21:57:57,125 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/46b5f83abe9585829b26884a1b26b43f/colfamily11/8fdc31e096a4428caf7a01bc018cbea5, entries=703, sequenceid=5, filesize=38.3 K 2024-11-15T21:57:57,129 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 46b5f83abe9585829b26884a1b26b43f in 95ms, sequenceid=5, compaction requested=false 2024-11-15T21:57:57,138 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/46b5f83abe9585829b26884a1b26b43f/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-15T21:57:57,140 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731707682333.46b5f83abe9585829b26884a1b26b43f. 2024-11-15T21:57:57,140 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 46b5f83abe9585829b26884a1b26b43f: Waiting for close lock at 1731707877034Running coprocessor pre-close hooks at 1731707877034Disabling compacts and flushes for region at 1731707877034Disabling writes for close at 1731707877034Obtaining lock to block concurrent updates at 1731707877034Preparing flush snapshotting stores in 46b5f83abe9585829b26884a1b26b43f at 1731707877034Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731707682333.46b5f83abe9585829b26884a1b26b43f., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731707877035 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731707682333.46b5f83abe9585829b26884a1b26b43f. at 1731707877040 (+5 ms)Flushing 46b5f83abe9585829b26884a1b26b43f/colfamily11: creating writer at 1731707877041 (+1 ms)Flushing 46b5f83abe9585829b26884a1b26b43f/colfamily11: appending metadata at 1731707877067 (+26 ms)Flushing 46b5f83abe9585829b26884a1b26b43f/colfamily11: closing flushed file at 1731707877067Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d41a040: reopening flushed file at 1731707877115 (+48 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 46b5f83abe9585829b26884a1b26b43f in 95ms, sequenceid=5, compaction requested=false at 1731707877129 (+14 ms)Writing region close event to WAL at 1731707877133 (+4 ms)Running coprocessor post-close hooks at 1731707877140 (+7 ms)Closed at 1731707877140 2024-11-15T21:57:57,140 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731707682333.46b5f83abe9585829b26884a1b26b43f. 2024-11-15T21:57:57,140 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 50df10b3ae47c5dc7c4521c4c58c192f, disabling compactions & flushes 2024-11-15T21:57:57,140 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731707682333.50df10b3ae47c5dc7c4521c4c58c192f. 2024-11-15T21:57:57,140 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731707682333.50df10b3ae47c5dc7c4521c4c58c192f. 2024-11-15T21:57:57,141 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731707682333.50df10b3ae47c5dc7c4521c4c58c192f. after waiting 0 ms 2024-11-15T21:57:57,141 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731707682333.50df10b3ae47c5dc7c4521c4c58c192f. 2024-11-15T21:57:57,141 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 50df10b3ae47c5dc7c4521c4c58c192f 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-15T21:57:57,142 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40941-0x100f86521fe0001, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T21:57:57,142 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40941-0x100f86521fe0001, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T21:57:57,145 INFO [RS:0;ae3311ea5ccb:40941 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-15T21:57:57,145 INFO [RS:0;ae3311ea5ccb:40941 {}] regionserver.HRegionServer(1031): Exiting; stopping=ae3311ea5ccb,40941,1731707679516; zookeeper connection closed. 2024-11-15T21:57:57,146 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7e4b586 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7e4b586 2024-11-15T21:57:57,168 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/50df10b3ae47c5dc7c4521c4c58c192f/.tmp/colfamily11/7332ef33fe454901ab73bc90da82682e is 52, key is ccc/colfamily11:colfamily11/1731707686512/Put/seqid=0 2024-11-15T21:57:57,186 DEBUG [RS:2;ae3311ea5ccb:37455 {}] regionserver.HRegionServer(1351): Waiting on 0890aeec1f0ca0649e6483a2d56f36b4, 50df10b3ae47c5dc7c4521c4c58c192f 2024-11-15T21:57:57,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741903_1079 (size=39263) 2024-11-15T21:57:57,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741903_1079 (size=39263) 2024-11-15T21:57:57,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741903_1079 (size=39263) 2024-11-15T21:57:57,386 DEBUG [RS:2;ae3311ea5ccb:37455 {}] regionserver.HRegionServer(1351): Waiting on 0890aeec1f0ca0649e6483a2d56f36b4, 50df10b3ae47c5dc7c4521c4c58c192f 2024-11-15T21:57:57,587 DEBUG [RS:2;ae3311ea5ccb:37455 {}] regionserver.HRegionServer(1351): Waiting on 0890aeec1f0ca0649e6483a2d56f36b4, 50df10b3ae47c5dc7c4521c4c58c192f 2024-11-15T21:57:57,591 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/50df10b3ae47c5dc7c4521c4c58c192f/.tmp/colfamily11/7332ef33fe454901ab73bc90da82682e 2024-11-15T21:57:57,600 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/50df10b3ae47c5dc7c4521c4c58c192f/.tmp/colfamily11/7332ef33fe454901ab73bc90da82682e as hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/50df10b3ae47c5dc7c4521c4c58c192f/colfamily11/7332ef33fe454901ab73bc90da82682e 2024-11-15T21:57:57,608 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/50df10b3ae47c5dc7c4521c4c58c192f/colfamily11/7332ef33fe454901ab73bc90da82682e, entries=703, sequenceid=5, filesize=38.3 K 2024-11-15T21:57:57,609 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 50df10b3ae47c5dc7c4521c4c58c192f in 468ms, sequenceid=5, compaction requested=false 2024-11-15T21:57:57,618 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/50df10b3ae47c5dc7c4521c4c58c192f/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-15T21:57:57,618 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731707682333.50df10b3ae47c5dc7c4521c4c58c192f. 2024-11-15T21:57:57,618 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 50df10b3ae47c5dc7c4521c4c58c192f: Waiting for close lock at 1731707877140Running coprocessor pre-close hooks at 1731707877140Disabling compacts and flushes for region at 1731707877140Disabling writes for close at 1731707877141 (+1 ms)Obtaining lock to block concurrent updates at 1731707877141Preparing flush snapshotting stores in 50df10b3ae47c5dc7c4521c4c58c192f at 1731707877141Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731707682333.50df10b3ae47c5dc7c4521c4c58c192f., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731707877141Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731707682333.50df10b3ae47c5dc7c4521c4c58c192f. at 1731707877143 (+2 ms)Flushing 50df10b3ae47c5dc7c4521c4c58c192f/colfamily11: creating writer at 1731707877143Flushing 50df10b3ae47c5dc7c4521c4c58c192f/colfamily11: appending metadata at 1731707877167 (+24 ms)Flushing 50df10b3ae47c5dc7c4521c4c58c192f/colfamily11: closing flushed file at 1731707877167Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f6fd73e: reopening flushed file at 1731707877599 (+432 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 50df10b3ae47c5dc7c4521c4c58c192f in 468ms, sequenceid=5, compaction requested=false at 1731707877609 (+10 ms)Writing region close event to WAL at 1731707877611 (+2 ms)Running coprocessor post-close hooks at 1731707877618 (+7 ms)Closed at 1731707877618 2024-11-15T21:57:57,619 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731707682333.50df10b3ae47c5dc7c4521c4c58c192f. 2024-11-15T21:57:57,619 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0890aeec1f0ca0649e6483a2d56f36b4, disabling compactions & flushes 2024-11-15T21:57:57,619 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731707682333.0890aeec1f0ca0649e6483a2d56f36b4. 2024-11-15T21:57:57,619 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731707682333.0890aeec1f0ca0649e6483a2d56f36b4. 2024-11-15T21:57:57,619 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731707682333.0890aeec1f0ca0649e6483a2d56f36b4. after waiting 0 ms 2024-11-15T21:57:57,619 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731707682333.0890aeec1f0ca0649e6483a2d56f36b4. 2024-11-15T21:57:57,619 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 0890aeec1f0ca0649e6483a2d56f36b4 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-15T21:57:57,639 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0890aeec1f0ca0649e6483a2d56f36b4/.tmp/colfamily11/1fe60f6cab86448f83b758562edf1d65 is 52, key is hhh/colfamily11:colfamily11/1731707686597/Put/seqid=0 2024-11-15T21:57:57,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741904_1080 (size=39263) 2024-11-15T21:57:57,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741904_1080 (size=39263) 2024-11-15T21:57:57,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741904_1080 (size=39263) 2024-11-15T21:57:57,679 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0890aeec1f0ca0649e6483a2d56f36b4/.tmp/colfamily11/1fe60f6cab86448f83b758562edf1d65 2024-11-15T21:57:57,695 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0890aeec1f0ca0649e6483a2d56f36b4/.tmp/colfamily11/1fe60f6cab86448f83b758562edf1d65 as hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0890aeec1f0ca0649e6483a2d56f36b4/colfamily11/1fe60f6cab86448f83b758562edf1d65 2024-11-15T21:57:57,705 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0890aeec1f0ca0649e6483a2d56f36b4/colfamily11/1fe60f6cab86448f83b758562edf1d65, entries=703, sequenceid=5, filesize=38.3 K 2024-11-15T21:57:57,707 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 0890aeec1f0ca0649e6483a2d56f36b4 in 88ms, sequenceid=5, compaction requested=false 2024-11-15T21:57:57,722 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0890aeec1f0ca0649e6483a2d56f36b4/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-15T21:57:57,723 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731707682333.0890aeec1f0ca0649e6483a2d56f36b4. 2024-11-15T21:57:57,723 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0890aeec1f0ca0649e6483a2d56f36b4: Waiting for close lock at 1731707877619Running coprocessor pre-close hooks at 1731707877619Disabling compacts and flushes for region at 1731707877619Disabling writes for close at 1731707877619Obtaining lock to block concurrent updates at 1731707877619Preparing flush snapshotting stores in 0890aeec1f0ca0649e6483a2d56f36b4 at 1731707877619Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731707682333.0890aeec1f0ca0649e6483a2d56f36b4., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731707877619Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731707682333.0890aeec1f0ca0649e6483a2d56f36b4. at 1731707877620 (+1 ms)Flushing 0890aeec1f0ca0649e6483a2d56f36b4/colfamily11: creating writer at 1731707877620Flushing 0890aeec1f0ca0649e6483a2d56f36b4/colfamily11: appending metadata at 1731707877639 (+19 ms)Flushing 0890aeec1f0ca0649e6483a2d56f36b4/colfamily11: closing flushed file at 1731707877639Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d1fd871: reopening flushed file at 1731707877694 (+55 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 0890aeec1f0ca0649e6483a2d56f36b4 in 88ms, sequenceid=5, compaction requested=false at 1731707877707 (+13 ms)Writing region close event to WAL at 1731707877714 (+7 ms)Running coprocessor post-close hooks at 1731707877722 (+8 ms)Closed at 1731707877722 2024-11-15T21:57:57,723 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731707682333.0890aeec1f0ca0649e6483a2d56f36b4. 2024-11-15T21:57:57,787 INFO [RS:2;ae3311ea5ccb:37455 {}] regionserver.HRegionServer(976): stopping server ae3311ea5ccb,37455,1731707679725; all regions closed. 2024-11-15T21:57:57,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741833_1009 (size=354137) 2024-11-15T21:57:57,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741833_1009 (size=354137) 2024-11-15T21:57:57,801 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/WALs/ae3311ea5ccb,37455,1731707679725/ae3311ea5ccb%2C37455%2C1731707679725.1731707681383 not finished, retry = 0 2024-11-15T21:57:57,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741833_1009 (size=354137) 2024-11-15T21:57:57,905 DEBUG [RS:2;ae3311ea5ccb:37455 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/oldWALs 2024-11-15T21:57:57,905 INFO [RS:2;ae3311ea5ccb:37455 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ae3311ea5ccb%2C37455%2C1731707679725:(num 1731707681383) 2024-11-15T21:57:57,905 DEBUG [RS:2;ae3311ea5ccb:37455 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T21:57:57,905 INFO [RS:2;ae3311ea5ccb:37455 {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T21:57:57,905 INFO [RS:2;ae3311ea5ccb:37455 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-15T21:57:57,906 INFO [RS:2;ae3311ea5ccb:37455 {}] hbase.ChoreService(370): Chore service for: regionserver/ae3311ea5ccb:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-15T21:57:57,906 INFO [RS:2;ae3311ea5ccb:37455 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-15T21:57:57,906 INFO [RS:2;ae3311ea5ccb:37455 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-15T21:57:57,906 INFO [RS:2;ae3311ea5ccb:37455 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-15T21:57:57,906 INFO [RS:2;ae3311ea5ccb:37455 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-15T21:57:57,906 INFO [RS:2;ae3311ea5ccb:37455 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:37455 2024-11-15T21:57:57,907 INFO [regionserver/ae3311ea5ccb:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-15T21:57:57,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37455-0x100f86521fe0003, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ae3311ea5ccb,37455,1731707679725 2024-11-15T21:57:57,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36531-0x100f86521fe0000, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T21:57:57,910 INFO [RS:2;ae3311ea5ccb:37455 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-15T21:57:57,911 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ae3311ea5ccb,37455,1731707679725] 2024-11-15T21:57:57,914 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ae3311ea5ccb,37455,1731707679725 already deleted, retry=false 2024-11-15T21:57:57,914 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ae3311ea5ccb,37455,1731707679725 expired; onlineServers=0 2024-11-15T21:57:57,914 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'ae3311ea5ccb,36531,1731707678636' ***** 2024-11-15T21:57:57,914 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-15T21:57:57,914 INFO [M:0;ae3311ea5ccb:36531 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-15T21:57:57,915 INFO [M:0;ae3311ea5ccb:36531 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-15T21:57:57,915 DEBUG [M:0;ae3311ea5ccb:36531 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-15T21:57:57,915 DEBUG [M:0;ae3311ea5ccb:36531 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-15T21:57:57,915 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-15T21:57:57,915 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster-HFileCleaner.small.0-1731707680948 {}] cleaner.HFileCleaner(306): Exit Thread[master/ae3311ea5ccb:0:becomeActiveMaster-HFileCleaner.small.0-1731707680948,5,FailOnTimeoutGroup] 2024-11-15T21:57:57,915 INFO [M:0;ae3311ea5ccb:36531 {}] hbase.ChoreService(370): Chore service for: master/ae3311ea5ccb:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-15T21:57:57,915 INFO [M:0;ae3311ea5ccb:36531 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-15T21:57:57,915 DEBUG [M:0;ae3311ea5ccb:36531 {}] master.HMaster(1795): Stopping service threads 2024-11-15T21:57:57,915 INFO [M:0;ae3311ea5ccb:36531 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-15T21:57:57,915 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster-HFileCleaner.large.0-1731707680947 {}] cleaner.HFileCleaner(306): Exit Thread[master/ae3311ea5ccb:0:becomeActiveMaster-HFileCleaner.large.0-1731707680947,5,FailOnTimeoutGroup] 2024-11-15T21:57:57,916 INFO [M:0;ae3311ea5ccb:36531 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T21:57:57,916 INFO [M:0;ae3311ea5ccb:36531 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T21:57:57,916 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-15T21:57:57,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36531-0x100f86521fe0000, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-15T21:57:57,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36531-0x100f86521fe0000, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:57:57,917 DEBUG [M:0;ae3311ea5ccb:36531 {}] zookeeper.ZKUtil(347): master:36531-0x100f86521fe0000, quorum=127.0.0.1:58023, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-15T21:57:57,918 WARN [M:0;ae3311ea5ccb:36531 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-15T21:57:57,918 INFO [M:0;ae3311ea5ccb:36531 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/.lastflushedseqids 2024-11-15T21:57:57,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741905_1081 (size=2837) 2024-11-15T21:57:57,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741905_1081 (size=2837) 2024-11-15T21:57:57,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741905_1081 (size=2837) 2024-11-15T21:57:57,985 INFO [M:0;ae3311ea5ccb:36531 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T21:57:57,985 INFO [M:0;ae3311ea5ccb:36531 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-15T21:57:57,985 DEBUG [M:0;ae3311ea5ccb:36531 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-15T21:57:57,985 INFO [M:0;ae3311ea5ccb:36531 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T21:57:57,985 DEBUG [M:0;ae3311ea5ccb:36531 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T21:57:57,985 DEBUG [M:0;ae3311ea5ccb:36531 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-15T21:57:57,985 DEBUG [M:0;ae3311ea5ccb:36531 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T21:57:57,986 INFO [M:0;ae3311ea5ccb:36531 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=344.65 KB heapSize=405.06 KB 2024-11-15T21:57:58,009 DEBUG [M:0;ae3311ea5ccb:36531 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/26a48e44c5a84815aa40aee31f29b638 is 82, key is hbase:meta,,1/info:regioninfo/1731707868843/Put/seqid=0 2024-11-15T21:57:58,013 INFO [RS:2;ae3311ea5ccb:37455 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-15T21:57:58,013 INFO [RS:2;ae3311ea5ccb:37455 {}] regionserver.HRegionServer(1031): Exiting; stopping=ae3311ea5ccb,37455,1731707679725; zookeeper connection closed. 2024-11-15T21:57:58,013 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37455-0x100f86521fe0003, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T21:57:58,013 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37455-0x100f86521fe0003, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T21:57:58,014 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6d5ad76b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6d5ad76b 2024-11-15T21:57:58,014 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-15T21:57:58,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741906_1082 (size=6071) 2024-11-15T21:57:58,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741906_1082 (size=6071) 2024-11-15T21:57:58,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741906_1082 (size=6071) 2024-11-15T21:57:58,062 INFO [M:0;ae3311ea5ccb:36531 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=635 (bloomFilter=true), to=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/26a48e44c5a84815aa40aee31f29b638 2024-11-15T21:57:58,091 DEBUG [M:0;ae3311ea5ccb:36531 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e1333c99a9d343fbaaa50e3fa6fc995d is 4105, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731707684298/Put/seqid=0 2024-11-15T21:57:58,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741907_1083 (size=13505) 2024-11-15T21:57:58,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741907_1083 (size=13505) 2024-11-15T21:57:58,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741907_1083 (size=13505) 2024-11-15T21:57:58,104 INFO [M:0;ae3311ea5ccb:36531 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=343.42 KB at sequenceid=635 (bloomFilter=true), to=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e1333c99a9d343fbaaa50e3fa6fc995d 2024-11-15T21:57:58,128 DEBUG [M:0;ae3311ea5ccb:36531 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/67bd62a31c5a4fb494a5b093854064d9 is 69, key is ae3311ea5ccb,37455,1731707679725/rs:state/1731707681068/Put/seqid=0 2024-11-15T21:57:58,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741908_1084 (size=5442) 2024-11-15T21:57:58,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741908_1084 (size=5442) 2024-11-15T21:57:58,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741908_1084 (size=5442) 2024-11-15T21:57:58,161 INFO [M:0;ae3311ea5ccb:36531 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=635 (bloomFilter=true), to=hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/67bd62a31c5a4fb494a5b093854064d9 2024-11-15T21:57:58,168 INFO [M:0;ae3311ea5ccb:36531 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 67bd62a31c5a4fb494a5b093854064d9 2024-11-15T21:57:58,170 DEBUG [M:0;ae3311ea5ccb:36531 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/26a48e44c5a84815aa40aee31f29b638 as hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/26a48e44c5a84815aa40aee31f29b638 2024-11-15T21:57:58,179 INFO [M:0;ae3311ea5ccb:36531 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/26a48e44c5a84815aa40aee31f29b638, entries=14, sequenceid=635, filesize=5.9 K 2024-11-15T21:57:58,181 DEBUG [M:0;ae3311ea5ccb:36531 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e1333c99a9d343fbaaa50e3fa6fc995d as hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e1333c99a9d343fbaaa50e3fa6fc995d 2024-11-15T21:57:58,195 INFO [M:0;ae3311ea5ccb:36531 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e1333c99a9d343fbaaa50e3fa6fc995d, entries=81, sequenceid=635, filesize=13.2 K 2024-11-15T21:57:58,198 DEBUG [M:0;ae3311ea5ccb:36531 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/67bd62a31c5a4fb494a5b093854064d9 as hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/67bd62a31c5a4fb494a5b093854064d9 2024-11-15T21:57:58,206 INFO [M:0;ae3311ea5ccb:36531 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 67bd62a31c5a4fb494a5b093854064d9 2024-11-15T21:57:58,206 INFO [M:0;ae3311ea5ccb:36531 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45565/user/jenkins/test-data/9e452d32-c547-9382-d829-c752b9d399ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/67bd62a31c5a4fb494a5b093854064d9, entries=3, sequenceid=635, filesize=5.3 K 2024-11-15T21:57:58,214 INFO [M:0;ae3311ea5ccb:36531 {}] regionserver.HRegion(3140): Finished flush of dataSize ~344.65 KB/352919, heapSize ~404.77 KB/414480, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 228ms, sequenceid=635, compaction requested=false 2024-11-15T21:57:58,228 INFO [M:0;ae3311ea5ccb:36531 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T21:57:58,228 DEBUG [M:0;ae3311ea5ccb:36531 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731707877985Disabling compacts and flushes for region at 1731707877985Disabling writes for close at 1731707877985Obtaining lock to block concurrent updates at 1731707877986 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731707877986Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=352919, getHeapSize=414720, getOffHeapSize=0, getCellsCount=800 at 1731707877986Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731707877987 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731707877987Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731707878009 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731707878009Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731707878072 (+63 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731707878090 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731707878090Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731707878110 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731707878127 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731707878127Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@45e4e3e1: reopening flushed file at 1731707878168 (+41 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4df5d664: reopening flushed file at 1731707878180 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@30780c9f: reopening flushed file at 1731707878196 (+16 ms)Finished flush of dataSize ~344.65 KB/352919, heapSize ~404.77 KB/414480, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 228ms, sequenceid=635, compaction requested=false at 1731707878214 (+18 ms)Writing region close event to WAL at 1731707878227 (+13 ms)Closed at 1731707878228 (+1 ms) 2024-11-15T21:57:58,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741830_1006 (size=395273) 2024-11-15T21:57:58,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36461 is added to blk_1073741830_1006 (size=395273) 2024-11-15T21:57:58,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40887 is added to blk_1073741830_1006 (size=395273) 2024-11-15T21:57:58,235 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-15T21:57:58,235 INFO [M:0;ae3311ea5ccb:36531 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-15T21:57:58,235 INFO [M:0;ae3311ea5ccb:36531 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:36531 2024-11-15T21:57:58,237 INFO [M:0;ae3311ea5ccb:36531 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-15T21:57:58,343 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36531-0x100f86521fe0000, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T21:57:58,343 INFO [M:0;ae3311ea5ccb:36531 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-15T21:57:58,343 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36531-0x100f86521fe0000, quorum=127.0.0.1:58023, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T21:57:58,352 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@22333f25{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-15T21:57:58,355 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@eec57c1{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-15T21:57:58,356 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-15T21:57:58,356 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@17888407{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-15T21:57:58,356 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@30067af9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ce7f090-fc0e-5e81-aacb-f56b26cecd72/hadoop.log.dir/,STOPPED} 2024-11-15T21:57:58,362 WARN [BP-1236864483-172.17.0.3-1731707675516 heartbeating to localhost/127.0.0.1:45565 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-15T21:57:58,362 WARN [BP-1236864483-172.17.0.3-1731707675516 heartbeating to localhost/127.0.0.1:45565 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1236864483-172.17.0.3-1731707675516 (Datanode Uuid 04ab36ac-20d1-483a-bb11-8fda764ca581) service to localhost/127.0.0.1:45565 2024-11-15T21:57:58,365 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-15T21:57:58,365 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-15T21:57:58,367 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ce7f090-fc0e-5e81-aacb-f56b26cecd72/cluster_370ef16d-d6cf-6544-0064-59de8e9843ef/data/data5/current/BP-1236864483-172.17.0.3-1731707675516 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-15T21:57:58,368 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-15T21:57:58,368 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ce7f090-fc0e-5e81-aacb-f56b26cecd72/cluster_370ef16d-d6cf-6544-0064-59de8e9843ef/data/data6/current/BP-1236864483-172.17.0.3-1731707675516 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-15T21:57:58,390 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1ba455be{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-15T21:57:58,395 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@259771a6{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-15T21:57:58,395 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-15T21:57:58,395 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6c516a28{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-15T21:57:58,395 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7f4017e5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ce7f090-fc0e-5e81-aacb-f56b26cecd72/hadoop.log.dir/,STOPPED} 2024-11-15T21:57:58,404 WARN [BP-1236864483-172.17.0.3-1731707675516 heartbeating to localhost/127.0.0.1:45565 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-15T21:57:58,404 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-15T21:57:58,405 WARN [BP-1236864483-172.17.0.3-1731707675516 heartbeating to localhost/127.0.0.1:45565 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1236864483-172.17.0.3-1731707675516 (Datanode Uuid 58e2f326-064e-4507-8270-f743dfbdcfa0) service to localhost/127.0.0.1:45565 2024-11-15T21:57:58,405 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-15T21:57:58,406 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ce7f090-fc0e-5e81-aacb-f56b26cecd72/cluster_370ef16d-d6cf-6544-0064-59de8e9843ef/data/data4/current/BP-1236864483-172.17.0.3-1731707675516 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-15T21:57:58,406 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-15T21:57:58,406 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ce7f090-fc0e-5e81-aacb-f56b26cecd72/cluster_370ef16d-d6cf-6544-0064-59de8e9843ef/data/data3/current/BP-1236864483-172.17.0.3-1731707675516 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-15T21:57:58,418 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4d81bf0e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-15T21:57:58,419 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@572fe942{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-15T21:57:58,419 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-15T21:57:58,419 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@664a7783{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-15T21:57:58,419 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7da31712{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ce7f090-fc0e-5e81-aacb-f56b26cecd72/hadoop.log.dir/,STOPPED} 2024-11-15T21:57:58,422 WARN [BP-1236864483-172.17.0.3-1731707675516 heartbeating to localhost/127.0.0.1:45565 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-15T21:57:58,422 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-15T21:57:58,422 WARN [BP-1236864483-172.17.0.3-1731707675516 heartbeating to localhost/127.0.0.1:45565 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1236864483-172.17.0.3-1731707675516 (Datanode Uuid 949a27da-6531-4c55-b5ce-f3af4b331818) service to localhost/127.0.0.1:45565 2024-11-15T21:57:58,422 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-15T21:57:58,422 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ce7f090-fc0e-5e81-aacb-f56b26cecd72/cluster_370ef16d-d6cf-6544-0064-59de8e9843ef/data/data1/current/BP-1236864483-172.17.0.3-1731707675516 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-15T21:57:58,423 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ce7f090-fc0e-5e81-aacb-f56b26cecd72/cluster_370ef16d-d6cf-6544-0064-59de8e9843ef/data/data2/current/BP-1236864483-172.17.0.3-1731707675516 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-15T21:57:58,423 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-15T21:57:58,440 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5183f126{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-15T21:57:58,441 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4350931a{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-15T21:57:58,441 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-15T21:57:58,441 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4974766f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-15T21:57:58,441 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2c447bb0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6ce7f090-fc0e-5e81-aacb-f56b26cecd72/hadoop.log.dir/,STOPPED} 2024-11-15T21:57:58,477 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-15T21:57:58,576 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-15T21:57:58,601 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.procedure.TestSCPWithMeta#testRecoveryAndDoubleExecutionOnRsWithMeta Thread=89 (was 11) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-6-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-7-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-6-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/ae3311ea5ccb:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: Abort regionserver monitor java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.2@localhost:45565 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: BootstrapNodeManager java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161) Potentially hanging thread: IPC Client (102774891) connection to localhost/127.0.0.1:45565 from jenkins.hfs.2 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: regionserver/ae3311ea5ccb:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: nioEventLoopGroup-4-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-5-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45565 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (102774891) connection to localhost/127.0.0.1:45565 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Client (102774891) connection to localhost/127.0.0.1:45565 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45565 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/ae3311ea5ccb:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: nioEventLoopGroup-7-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45565 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@492dea64 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253) app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-7-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/ae3311ea5ccb:0.Chore.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (102774891) connection to localhost/127.0.0.1:45565 from jenkins.hfs.0 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-5-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/ae3311ea5ccb:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-6-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/ae3311ea5ccb:0.logRoller java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:180) Potentially hanging thread: ForkJoinPool-2-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: master/ae3311ea5ccb:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-5-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SSL Certificates Store Monitor java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: LeaseRenewer:jenkins.hfs.0@localhost:45565 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:45565 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45565 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=456 (was 285) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=479 (was 267) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1154 (was 3628)