2024-11-18 18:24:11,937 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-18 18:24:11,957 main DEBUG Took 0.017605 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-18 18:24:11,957 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-18 18:24:11,958 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-18 18:24:11,958 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-18 18:24:11,960 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 18:24:11,967 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-18 18:24:11,985 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 18:24:11,987 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 18:24:11,988 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 18:24:11,988 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 18:24:11,990 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 18:24:11,991 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 18:24:11,993 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 18:24:11,993 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 18:24:11,994 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 18:24:11,994 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 18:24:11,995 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 18:24:11,996 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 18:24:11,996 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 18:24:11,997 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 18:24:11,998 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 18:24:11,998 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 18:24:11,999 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 18:24:11,999 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 18:24:12,000 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 18:24:12,000 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 18:24:12,001 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 18:24:12,001 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 18:24:12,002 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 18:24:12,002 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 18:24:12,003 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 18:24:12,003 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-18 18:24:12,005 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 18:24:12,007 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-18 18:24:12,009 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-18 18:24:12,010 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-18 18:24:12,012 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-18 18:24:12,012 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-18 18:24:12,024 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-18 18:24:12,027 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-18 18:24:12,030 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-18 18:24:12,030 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-18 18:24:12,031 main DEBUG createAppenders(={Console}) 2024-11-18 18:24:12,032 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-18 18:24:12,032 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-18 18:24:12,033 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-18 18:24:12,033 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-18 18:24:12,034 main DEBUG OutputStream closed 2024-11-18 18:24:12,034 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-18 18:24:12,034 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-18 18:24:12,035 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-18 18:24:12,124 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-18 18:24:12,127 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-18 18:24:12,129 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-18 18:24:12,130 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-18 18:24:12,130 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-18 18:24:12,131 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-18 18:24:12,131 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-18 18:24:12,131 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-18 18:24:12,132 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-18 18:24:12,132 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-18 18:24:12,132 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-18 18:24:12,132 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-18 18:24:12,133 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-18 18:24:12,133 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-18 18:24:12,133 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-18 18:24:12,134 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-18 18:24:12,134 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-18 18:24:12,135 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-18 18:24:12,137 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-18 18:24:12,137 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-18 18:24:12,138 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-18 18:24:12,139 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-18T18:24:12,158 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.procedure.TestSCPWithMeta timeout: 13 mins 2024-11-18 18:24:12,161 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-18 18:24:12,162 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-18T18:24:12,203 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.procedure.TestSCPWithMeta#testRecoveryAndDoubleExecutionOnRsWithMeta Thread=11, OpenFileDescriptor=285, MaxFileDescriptor=1048576, SystemLoadAverage=607, ProcessCount=11, AvailableMemoryMB=3808 2024-11-18T18:24:12,512 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67b3225c-0ec5-86bb-7544-cb0f1a3b69d6 2024-11-18T18:24:12,513 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-18T18:24:12,533 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67b3225c-0ec5-86bb-7544-cb0f1a3b69d6/cluster_cb6102b6-aad9-a0f9-bd1c-aa9450167237, deleteOnExit=true 2024-11-18T18:24:12,534 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-18T18:24:12,535 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67b3225c-0ec5-86bb-7544-cb0f1a3b69d6/test.cache.data in system properties and HBase conf 2024-11-18T18:24:12,536 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67b3225c-0ec5-86bb-7544-cb0f1a3b69d6/hadoop.tmp.dir in system properties and HBase conf 2024-11-18T18:24:12,537 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67b3225c-0ec5-86bb-7544-cb0f1a3b69d6/hadoop.log.dir in system properties and HBase conf 2024-11-18T18:24:12,537 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67b3225c-0ec5-86bb-7544-cb0f1a3b69d6/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-18T18:24:12,538 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67b3225c-0ec5-86bb-7544-cb0f1a3b69d6/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-18T18:24:12,539 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-18T18:24:12,651 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-18T18:24:12,765 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T18:24:12,769 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67b3225c-0ec5-86bb-7544-cb0f1a3b69d6/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-18T18:24:12,770 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67b3225c-0ec5-86bb-7544-cb0f1a3b69d6/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-18T18:24:12,771 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67b3225c-0ec5-86bb-7544-cb0f1a3b69d6/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-18T18:24:12,771 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67b3225c-0ec5-86bb-7544-cb0f1a3b69d6/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-18T18:24:12,772 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67b3225c-0ec5-86bb-7544-cb0f1a3b69d6/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-18T18:24:12,772 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67b3225c-0ec5-86bb-7544-cb0f1a3b69d6/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-18T18:24:12,772 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67b3225c-0ec5-86bb-7544-cb0f1a3b69d6/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-18T18:24:12,773 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67b3225c-0ec5-86bb-7544-cb0f1a3b69d6/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-18T18:24:12,773 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67b3225c-0ec5-86bb-7544-cb0f1a3b69d6/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-18T18:24:12,774 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67b3225c-0ec5-86bb-7544-cb0f1a3b69d6/nfs.dump.dir in system properties and HBase conf 2024-11-18T18:24:12,774 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67b3225c-0ec5-86bb-7544-cb0f1a3b69d6/java.io.tmpdir in system properties and HBase conf 2024-11-18T18:24:12,774 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67b3225c-0ec5-86bb-7544-cb0f1a3b69d6/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-18T18:24:12,775 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67b3225c-0ec5-86bb-7544-cb0f1a3b69d6/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-18T18:24:12,775 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67b3225c-0ec5-86bb-7544-cb0f1a3b69d6/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-18T18:24:13,675 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-18T18:24:13,770 INFO [Time-limited test {}] log.Log(170): Logging initialized @2751ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-18T18:24:13,867 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-18T18:24:13,958 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-18T18:24:14,000 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-18T18:24:14,000 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-18T18:24:14,003 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-18T18:24:14,021 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-18T18:24:14,025 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2c447bb0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67b3225c-0ec5-86bb-7544-cb0f1a3b69d6/hadoop.log.dir/,AVAILABLE} 2024-11-18T18:24:14,026 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4974766f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-18T18:24:14,299 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5183f126{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67b3225c-0ec5-86bb-7544-cb0f1a3b69d6/java.io.tmpdir/jetty-localhost-36859-hadoop-hdfs-3_4_1-tests_jar-_-any-14241327103190578026/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-18T18:24:14,308 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4350931a{HTTP/1.1, (http/1.1)}{localhost:36859} 2024-11-18T18:24:14,309 INFO [Time-limited test {}] server.Server(415): Started @3291ms 2024-11-18T18:24:14,752 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-18T18:24:14,763 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-18T18:24:14,765 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-18T18:24:14,766 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-18T18:24:14,766 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-18T18:24:14,767 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@463823fd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67b3225c-0ec5-86bb-7544-cb0f1a3b69d6/hadoop.log.dir/,AVAILABLE} 2024-11-18T18:24:14,767 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93e0721{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-18T18:24:14,891 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6f328155{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67b3225c-0ec5-86bb-7544-cb0f1a3b69d6/java.io.tmpdir/jetty-localhost-36329-hadoop-hdfs-3_4_1-tests_jar-_-any-9542383426084485663/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-18T18:24:14,892 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6fd8ef10{HTTP/1.1, (http/1.1)}{localhost:36329} 2024-11-18T18:24:14,892 INFO [Time-limited test {}] server.Server(415): Started @3874ms 2024-11-18T18:24:14,954 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-18T18:24:15,105 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-18T18:24:15,118 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-18T18:24:15,130 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-18T18:24:15,130 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-18T18:24:15,131 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-18T18:24:15,131 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@57794ee2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67b3225c-0ec5-86bb-7544-cb0f1a3b69d6/hadoop.log.dir/,AVAILABLE} 2024-11-18T18:24:15,132 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7186ce9f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-18T18:24:15,297 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@666ae17c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67b3225c-0ec5-86bb-7544-cb0f1a3b69d6/java.io.tmpdir/jetty-localhost-36877-hadoop-hdfs-3_4_1-tests_jar-_-any-15821466000808913258/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-18T18:24:15,298 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@270be46{HTTP/1.1, (http/1.1)}{localhost:36877} 2024-11-18T18:24:15,298 INFO [Time-limited test {}] server.Server(415): Started @4280ms 2024-11-18T18:24:15,301 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-18T18:24:15,349 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-18T18:24:15,355 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-18T18:24:15,371 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-18T18:24:15,372 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-18T18:24:15,372 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-18T18:24:15,374 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@56f216fb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67b3225c-0ec5-86bb-7544-cb0f1a3b69d6/hadoop.log.dir/,AVAILABLE} 2024-11-18T18:24:15,378 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7f9dc0ae{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-18T18:24:15,490 WARN [Thread-105 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67b3225c-0ec5-86bb-7544-cb0f1a3b69d6/cluster_cb6102b6-aad9-a0f9-bd1c-aa9450167237/data/data1/current/BP-868939734-172.17.0.2-1731954253414/current, will proceed with Du for space computation calculation, 2024-11-18T18:24:15,490 WARN [Thread-107 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67b3225c-0ec5-86bb-7544-cb0f1a3b69d6/cluster_cb6102b6-aad9-a0f9-bd1c-aa9450167237/data/data2/current/BP-868939734-172.17.0.2-1731954253414/current, will proceed with Du for space computation calculation, 2024-11-18T18:24:15,491 WARN [Thread-106 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67b3225c-0ec5-86bb-7544-cb0f1a3b69d6/cluster_cb6102b6-aad9-a0f9-bd1c-aa9450167237/data/data3/current/BP-868939734-172.17.0.2-1731954253414/current, will proceed with Du for space computation calculation, 2024-11-18T18:24:15,493 WARN [Thread-108 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67b3225c-0ec5-86bb-7544-cb0f1a3b69d6/cluster_cb6102b6-aad9-a0f9-bd1c-aa9450167237/data/data4/current/BP-868939734-172.17.0.2-1731954253414/current, will proceed with Du for space computation calculation, 2024-11-18T18:24:15,568 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-18T18:24:15,578 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6478847d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67b3225c-0ec5-86bb-7544-cb0f1a3b69d6/java.io.tmpdir/jetty-localhost-44983-hadoop-hdfs-3_4_1-tests_jar-_-any-6681640968216171642/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-18T18:24:15,580 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@15e91890{HTTP/1.1, (http/1.1)}{localhost:44983} 2024-11-18T18:24:15,581 INFO [Time-limited test {}] server.Server(415): Started @4563ms 2024-11-18T18:24:15,587 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-18T18:24:15,598 WARN [Thread-82 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-18T18:24:15,661 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8e9d43e1b6449a0b with lease ID 0x450a4f549163a5c1: Processing first storage report for DS-39ff43bf-c18b-4b86-b37c-eca99a433383 from datanode DatanodeRegistration(127.0.0.1:44191, datanodeUuid=76d78cf9-913a-4743-9820-1225728dbf25, infoPort=34389, infoSecurePort=0, ipcPort=46715, storageInfo=lv=-57;cid=testClusterID;nsid=25584234;c=1731954253414) 2024-11-18T18:24:15,662 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8e9d43e1b6449a0b with lease ID 0x450a4f549163a5c1: from storage DS-39ff43bf-c18b-4b86-b37c-eca99a433383 node DatanodeRegistration(127.0.0.1:44191, datanodeUuid=76d78cf9-913a-4743-9820-1225728dbf25, infoPort=34389, infoSecurePort=0, ipcPort=46715, storageInfo=lv=-57;cid=testClusterID;nsid=25584234;c=1731954253414), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-18T18:24:15,663 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x635e7450ab4bcb35 with lease ID 0x450a4f549163a5c2: Processing first storage report for DS-c75b1747-362b-431f-b47d-543a4f882cdb from datanode DatanodeRegistration(127.0.0.1:41713, datanodeUuid=5a9c9d5e-ee42-47fc-ab8f-e8ed647e5b1c, infoPort=41643, infoSecurePort=0, ipcPort=45135, storageInfo=lv=-57;cid=testClusterID;nsid=25584234;c=1731954253414) 2024-11-18T18:24:15,663 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x635e7450ab4bcb35 with lease ID 0x450a4f549163a5c2: from storage DS-c75b1747-362b-431f-b47d-543a4f882cdb node DatanodeRegistration(127.0.0.1:41713, datanodeUuid=5a9c9d5e-ee42-47fc-ab8f-e8ed647e5b1c, infoPort=41643, infoSecurePort=0, ipcPort=45135, storageInfo=lv=-57;cid=testClusterID;nsid=25584234;c=1731954253414), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-18T18:24:15,663 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8e9d43e1b6449a0b with lease ID 0x450a4f549163a5c1: Processing first storage report for DS-ca412d02-9703-4e38-9066-8690d4f32976 from datanode DatanodeRegistration(127.0.0.1:44191, datanodeUuid=76d78cf9-913a-4743-9820-1225728dbf25, infoPort=34389, infoSecurePort=0, ipcPort=46715, storageInfo=lv=-57;cid=testClusterID;nsid=25584234;c=1731954253414) 2024-11-18T18:24:15,663 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8e9d43e1b6449a0b with lease ID 0x450a4f549163a5c1: from storage DS-ca412d02-9703-4e38-9066-8690d4f32976 node DatanodeRegistration(127.0.0.1:44191, datanodeUuid=76d78cf9-913a-4743-9820-1225728dbf25, infoPort=34389, infoSecurePort=0, ipcPort=46715, storageInfo=lv=-57;cid=testClusterID;nsid=25584234;c=1731954253414), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-18T18:24:15,663 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x635e7450ab4bcb35 with lease ID 0x450a4f549163a5c2: Processing first storage report for DS-1445370b-e50a-4001-91aa-73b666b1eb2c from datanode DatanodeRegistration(127.0.0.1:41713, datanodeUuid=5a9c9d5e-ee42-47fc-ab8f-e8ed647e5b1c, infoPort=41643, infoSecurePort=0, ipcPort=45135, storageInfo=lv=-57;cid=testClusterID;nsid=25584234;c=1731954253414) 2024-11-18T18:24:15,664 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x635e7450ab4bcb35 with lease ID 0x450a4f549163a5c2: from storage DS-1445370b-e50a-4001-91aa-73b666b1eb2c node DatanodeRegistration(127.0.0.1:41713, datanodeUuid=5a9c9d5e-ee42-47fc-ab8f-e8ed647e5b1c, infoPort=41643, infoSecurePort=0, ipcPort=45135, storageInfo=lv=-57;cid=testClusterID;nsid=25584234;c=1731954253414), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-18T18:24:15,758 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67b3225c-0ec5-86bb-7544-cb0f1a3b69d6/cluster_cb6102b6-aad9-a0f9-bd1c-aa9450167237/data/data6/current/BP-868939734-172.17.0.2-1731954253414/current, will proceed with Du for space computation calculation, 2024-11-18T18:24:15,759 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67b3225c-0ec5-86bb-7544-cb0f1a3b69d6/cluster_cb6102b6-aad9-a0f9-bd1c-aa9450167237/data/data5/current/BP-868939734-172.17.0.2-1731954253414/current, will proceed with Du for space computation calculation, 2024-11-18T18:24:15,814 WARN [Thread-129 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-18T18:24:15,823 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x664a2420134de211 with lease ID 0x450a4f549163a5c3: Processing first storage report for DS-1029886b-6b51-4ae4-81a3-156afc7de901 from datanode DatanodeRegistration(127.0.0.1:34047, datanodeUuid=7f99a8ef-4a0d-4d55-b1dd-487bc0910b13, infoPort=38097, infoSecurePort=0, ipcPort=44109, storageInfo=lv=-57;cid=testClusterID;nsid=25584234;c=1731954253414) 2024-11-18T18:24:15,823 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x664a2420134de211 with lease ID 0x450a4f549163a5c3: from storage DS-1029886b-6b51-4ae4-81a3-156afc7de901 node DatanodeRegistration(127.0.0.1:34047, datanodeUuid=7f99a8ef-4a0d-4d55-b1dd-487bc0910b13, infoPort=38097, infoSecurePort=0, ipcPort=44109, storageInfo=lv=-57;cid=testClusterID;nsid=25584234;c=1731954253414), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-18T18:24:15,824 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x664a2420134de211 with lease ID 0x450a4f549163a5c3: Processing first storage report for DS-8fd70858-20db-4ebd-bc84-90901e0e8ac0 from datanode DatanodeRegistration(127.0.0.1:34047, datanodeUuid=7f99a8ef-4a0d-4d55-b1dd-487bc0910b13, infoPort=38097, infoSecurePort=0, ipcPort=44109, storageInfo=lv=-57;cid=testClusterID;nsid=25584234;c=1731954253414) 2024-11-18T18:24:15,824 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x664a2420134de211 with lease ID 0x450a4f549163a5c3: from storage DS-8fd70858-20db-4ebd-bc84-90901e0e8ac0 node DatanodeRegistration(127.0.0.1:34047, datanodeUuid=7f99a8ef-4a0d-4d55-b1dd-487bc0910b13, infoPort=38097, infoSecurePort=0, ipcPort=44109, storageInfo=lv=-57;cid=testClusterID;nsid=25584234;c=1731954253414), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-18T18:24:16,054 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67b3225c-0ec5-86bb-7544-cb0f1a3b69d6 2024-11-18T18:24:16,136 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67b3225c-0ec5-86bb-7544-cb0f1a3b69d6/cluster_cb6102b6-aad9-a0f9-bd1c-aa9450167237/zookeeper_0, clientPort=59292, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67b3225c-0ec5-86bb-7544-cb0f1a3b69d6/cluster_cb6102b6-aad9-a0f9-bd1c-aa9450167237/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67b3225c-0ec5-86bb-7544-cb0f1a3b69d6/cluster_cb6102b6-aad9-a0f9-bd1c-aa9450167237/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-18T18:24:16,148 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59292 2024-11-18T18:24:16,160 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T18:24:16,164 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T18:24:16,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741825_1001 (size=7) 2024-11-18T18:24:16,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741825_1001 (size=7) 2024-11-18T18:24:16,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741825_1001 (size=7) 2024-11-18T18:24:16,846 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93 with version=8 2024-11-18T18:24:16,847 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/hbase-staging 2024-11-18T18:24:16,944 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-18T18:24:17,202 INFO [Time-limited test {}] client.ConnectionUtils(128): master/961a81da9827:0 server-side Connection retries=45 2024-11-18T18:24:17,212 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T18:24:17,213 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T18:24:17,217 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T18:24:17,218 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T18:24:17,218 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T18:24:17,368 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-18T18:24:17,434 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-18T18:24:17,445 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-18T18:24:17,450 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T18:24:17,481 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 15975 (auto-detected) 2024-11-18T18:24:17,482 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-18T18:24:17,501 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33089 2024-11-18T18:24:17,531 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33089 connecting to ZooKeeper ensemble=127.0.0.1:59292 2024-11-18T18:24:17,565 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:330890x0, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T18:24:17,567 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33089-0x10107179d9f0000 connected 2024-11-18T18:24:17,595 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T18:24:17,600 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T18:24:17,614 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33089-0x10107179d9f0000, quorum=127.0.0.1:59292, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T18:24:17,619 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93, hbase.cluster.distributed=false 2024-11-18T18:24:17,642 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33089-0x10107179d9f0000, quorum=127.0.0.1:59292, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T18:24:17,647 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33089 2024-11-18T18:24:17,647 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33089 2024-11-18T18:24:17,649 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33089 2024-11-18T18:24:17,649 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33089 2024-11-18T18:24:17,650 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33089 2024-11-18T18:24:17,766 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/961a81da9827:0 server-side Connection retries=45 2024-11-18T18:24:17,768 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T18:24:17,768 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T18:24:17,768 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T18:24:17,768 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T18:24:17,768 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T18:24:17,771 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-18T18:24:17,773 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T18:24:17,774 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38709 2024-11-18T18:24:17,776 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38709 connecting to ZooKeeper ensemble=127.0.0.1:59292 2024-11-18T18:24:17,777 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T18:24:17,779 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T18:24:17,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:387090x0, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T18:24:17,786 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38709-0x10107179d9f0001 connected 2024-11-18T18:24:17,786 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38709-0x10107179d9f0001, quorum=127.0.0.1:59292, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T18:24:17,790 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-18T18:24:17,798 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-18T18:24:17,800 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38709-0x10107179d9f0001, quorum=127.0.0.1:59292, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-18T18:24:17,805 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38709-0x10107179d9f0001, quorum=127.0.0.1:59292, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T18:24:17,806 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38709 2024-11-18T18:24:17,806 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38709 2024-11-18T18:24:17,807 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38709 2024-11-18T18:24:17,808 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38709 2024-11-18T18:24:17,808 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38709 2024-11-18T18:24:17,826 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/961a81da9827:0 server-side Connection retries=45 2024-11-18T18:24:17,826 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T18:24:17,827 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T18:24:17,827 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T18:24:17,827 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T18:24:17,827 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T18:24:17,828 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-18T18:24:17,828 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T18:24:17,829 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36379 2024-11-18T18:24:17,831 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36379 connecting to ZooKeeper ensemble=127.0.0.1:59292 2024-11-18T18:24:17,831 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T18:24:17,834 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T18:24:17,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:363790x0, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T18:24:17,840 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36379-0x10107179d9f0002 connected 2024-11-18T18:24:17,840 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36379-0x10107179d9f0002, quorum=127.0.0.1:59292, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T18:24:17,841 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-18T18:24:17,842 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-18T18:24:17,843 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36379-0x10107179d9f0002, quorum=127.0.0.1:59292, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-18T18:24:17,844 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36379-0x10107179d9f0002, quorum=127.0.0.1:59292, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T18:24:17,845 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36379 2024-11-18T18:24:17,846 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36379 2024-11-18T18:24:17,848 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36379 2024-11-18T18:24:17,849 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36379 2024-11-18T18:24:17,849 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36379 2024-11-18T18:24:17,867 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/961a81da9827:0 server-side Connection retries=45 2024-11-18T18:24:17,867 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T18:24:17,868 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T18:24:17,868 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T18:24:17,868 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T18:24:17,868 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T18:24:17,868 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-18T18:24:17,868 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T18:24:17,869 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42895 2024-11-18T18:24:17,871 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42895 connecting to ZooKeeper ensemble=127.0.0.1:59292 2024-11-18T18:24:17,872 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T18:24:17,874 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T18:24:17,880 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:428950x0, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T18:24:17,880 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42895-0x10107179d9f0003 connected 2024-11-18T18:24:17,881 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42895-0x10107179d9f0003, quorum=127.0.0.1:59292, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T18:24:17,881 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-18T18:24:17,883 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-18T18:24:17,884 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42895-0x10107179d9f0003, quorum=127.0.0.1:59292, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-18T18:24:17,887 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42895-0x10107179d9f0003, quorum=127.0.0.1:59292, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T18:24:17,890 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42895 2024-11-18T18:24:17,893 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42895 2024-11-18T18:24:17,896 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42895 2024-11-18T18:24:17,898 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42895 2024-11-18T18:24:17,898 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42895 2024-11-18T18:24:17,916 DEBUG [M:0;961a81da9827:33089 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;961a81da9827:33089 2024-11-18T18:24:17,917 INFO [master/961a81da9827:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/961a81da9827,33089,1731954257001 2024-11-18T18:24:17,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36379-0x10107179d9f0002, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T18:24:17,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33089-0x10107179d9f0000, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T18:24:17,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38709-0x10107179d9f0001, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T18:24:17,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42895-0x10107179d9f0003, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T18:24:17,928 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33089-0x10107179d9f0000, quorum=127.0.0.1:59292, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/961a81da9827,33089,1731954257001 2024-11-18T18:24:17,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42895-0x10107179d9f0003, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-18T18:24:17,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36379-0x10107179d9f0002, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-18T18:24:17,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38709-0x10107179d9f0001, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-18T18:24:17,958 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33089-0x10107179d9f0000, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:24:17,958 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38709-0x10107179d9f0001, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:24:17,958 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42895-0x10107179d9f0003, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:24:17,958 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36379-0x10107179d9f0002, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:24:17,960 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33089-0x10107179d9f0000, quorum=127.0.0.1:59292, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-18T18:24:17,961 INFO [master/961a81da9827:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/961a81da9827,33089,1731954257001 from backup master directory 2024-11-18T18:24:17,965 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33089-0x10107179d9f0000, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/961a81da9827,33089,1731954257001 2024-11-18T18:24:17,966 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38709-0x10107179d9f0001, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T18:24:17,966 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33089-0x10107179d9f0000, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T18:24:17,966 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42895-0x10107179d9f0003, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T18:24:17,966 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36379-0x10107179d9f0002, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T18:24:17,967 WARN [master/961a81da9827:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T18:24:17,967 INFO [master/961a81da9827:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=961a81da9827,33089,1731954257001 2024-11-18T18:24:17,970 INFO [master/961a81da9827:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-18T18:24:17,972 INFO [master/961a81da9827:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-18T18:24:18,030 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/hbase.id] with ID: d961f11e-5c15-416e-a960-3926a0e53aca 2024-11-18T18:24:18,030 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/.tmp/hbase.id 2024-11-18T18:24:18,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741826_1002 (size=42) 2024-11-18T18:24:18,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741826_1002 (size=42) 2024-11-18T18:24:18,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741826_1002 (size=42) 2024-11-18T18:24:18,047 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/.tmp/hbase.id]:[hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/hbase.id] 2024-11-18T18:24:18,091 INFO [master/961a81da9827:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T18:24:18,096 INFO [master/961a81da9827:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-18T18:24:18,118 INFO [master/961a81da9827:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-18T18:24:18,122 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38709-0x10107179d9f0001, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:24:18,122 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36379-0x10107179d9f0002, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:24:18,122 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42895-0x10107179d9f0003, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:24:18,123 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33089-0x10107179d9f0000, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:24:18,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741827_1003 (size=196) 2024-11-18T18:24:18,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741827_1003 (size=196) 2024-11-18T18:24:18,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741827_1003 (size=196) 2024-11-18T18:24:18,164 INFO [master/961a81da9827:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T18:24:18,165 INFO [master/961a81da9827:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-18T18:24:18,183 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T18:24:18,188 INFO [master/961a81da9827:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T18:24:18,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741828_1004 (size=1189) 2024-11-18T18:24:18,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741828_1004 (size=1189) 2024-11-18T18:24:18,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741828_1004 (size=1189) 2024-11-18T18:24:18,244 INFO [master/961a81da9827:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/MasterData/data/master/store 2024-11-18T18:24:18,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741829_1005 (size=34) 2024-11-18T18:24:18,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741829_1005 (size=34) 2024-11-18T18:24:18,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741829_1005 (size=34) 2024-11-18T18:24:18,275 INFO [master/961a81da9827:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-18T18:24:18,278 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:18,279 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-18T18:24:18,279 INFO [master/961a81da9827:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T18:24:18,280 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T18:24:18,281 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-18T18:24:18,281 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T18:24:18,282 INFO [master/961a81da9827:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T18:24:18,283 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731954258279Disabling compacts and flushes for region at 1731954258279Disabling writes for close at 1731954258281 (+2 ms)Writing region close event to WAL at 1731954258282 (+1 ms)Closed at 1731954258282 2024-11-18T18:24:18,285 WARN [master/961a81da9827:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/MasterData/data/master/store/.initializing 2024-11-18T18:24:18,285 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/MasterData/WALs/961a81da9827,33089,1731954257001 2024-11-18T18:24:18,294 INFO [master/961a81da9827:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T18:24:18,308 INFO [master/961a81da9827:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=961a81da9827%2C33089%2C1731954257001, suffix=, logDir=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/MasterData/WALs/961a81da9827,33089,1731954257001, archiveDir=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/MasterData/oldWALs, maxLogs=10 2024-11-18T18:24:18,333 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/MasterData/WALs/961a81da9827,33089,1731954257001/961a81da9827%2C33089%2C1731954257001.1731954258313, exclude list is [], retry=0 2024-11-18T18:24:18,351 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34047,DS-1029886b-6b51-4ae4-81a3-156afc7de901,DISK] 2024-11-18T18:24:18,351 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41713,DS-c75b1747-362b-431f-b47d-543a4f882cdb,DISK] 2024-11-18T18:24:18,351 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44191,DS-39ff43bf-c18b-4b86-b37c-eca99a433383,DISK] 2024-11-18T18:24:18,355 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-18T18:24:18,396 INFO [master/961a81da9827:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/MasterData/WALs/961a81da9827,33089,1731954257001/961a81da9827%2C33089%2C1731954257001.1731954258313 2024-11-18T18:24:18,397 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41643:41643),(127.0.0.1/127.0.0.1:34389:34389),(127.0.0.1/127.0.0.1:38097:38097)] 2024-11-18T18:24:18,397 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-18T18:24:18,398 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:18,402 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T18:24:18,403 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T18:24:18,442 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T18:24:18,469 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-18T18:24:18,473 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:18,475 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T18:24:18,476 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T18:24:18,479 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-18T18:24:18,479 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:18,480 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:18,480 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T18:24:18,483 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-18T18:24:18,483 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:18,484 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:18,484 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T18:24:18,487 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-18T18:24:18,487 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:18,488 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:18,488 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T18:24:18,491 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-18T18:24:18,492 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-18T18:24:18,497 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T18:24:18,498 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T18:24:18,501 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T18:24:18,505 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T18:24:18,512 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T18:24:18,514 INFO [master/961a81da9827:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68665139, jitterRate=0.02319030463695526}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T18:24:18,520 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731954258416Initializing all the Stores at 1731954258418 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731954258419 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954258419Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954258419Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954258419Cleaning up temporary data from old regions at 1731954258498 (+79 ms)Region opened successfully at 1731954258519 (+21 ms) 2024-11-18T18:24:18,521 INFO [master/961a81da9827:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-18T18:24:18,559 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3db668ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=961a81da9827/172.17.0.2:0 2024-11-18T18:24:18,592 INFO [master/961a81da9827:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-18T18:24:18,605 INFO [master/961a81da9827:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-18T18:24:18,605 INFO [master/961a81da9827:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-18T18:24:18,609 INFO [master/961a81da9827:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-18T18:24:18,610 INFO [master/961a81da9827:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-18T18:24:18,616 INFO [master/961a81da9827:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-18T18:24:18,616 INFO [master/961a81da9827:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-18T18:24:18,644 INFO [master/961a81da9827:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-18T18:24:18,652 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33089-0x10107179d9f0000, quorum=127.0.0.1:59292, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-18T18:24:18,654 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-18T18:24:18,658 INFO [master/961a81da9827:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-18T18:24:18,660 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33089-0x10107179d9f0000, quorum=127.0.0.1:59292, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-18T18:24:18,661 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-18T18:24:18,664 INFO [master/961a81da9827:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-18T18:24:18,669 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33089-0x10107179d9f0000, quorum=127.0.0.1:59292, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-18T18:24:18,671 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-18T18:24:18,672 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33089-0x10107179d9f0000, quorum=127.0.0.1:59292, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-18T18:24:18,674 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-18T18:24:18,694 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33089-0x10107179d9f0000, quorum=127.0.0.1:59292, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-18T18:24:18,696 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-18T18:24:18,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33089-0x10107179d9f0000, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T18:24:18,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38709-0x10107179d9f0001, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T18:24:18,701 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38709-0x10107179d9f0001, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:24:18,701 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33089-0x10107179d9f0000, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:24:18,701 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36379-0x10107179d9f0002, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T18:24:18,701 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42895-0x10107179d9f0003, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T18:24:18,701 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36379-0x10107179d9f0002, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:24:18,701 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42895-0x10107179d9f0003, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:24:18,704 INFO [master/961a81da9827:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=961a81da9827,33089,1731954257001, sessionid=0x10107179d9f0000, setting cluster-up flag (Was=false) 2024-11-18T18:24:18,718 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36379-0x10107179d9f0002, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:24:18,718 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42895-0x10107179d9f0003, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:24:18,718 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38709-0x10107179d9f0001, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:24:18,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33089-0x10107179d9f0000, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:24:18,725 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-18T18:24:18,727 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=961a81da9827,33089,1731954257001 2024-11-18T18:24:18,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33089-0x10107179d9f0000, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:24:18,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38709-0x10107179d9f0001, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:24:18,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42895-0x10107179d9f0003, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:24:18,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36379-0x10107179d9f0002, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:24:18,740 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-18T18:24:18,742 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=961a81da9827,33089,1731954257001 2024-11-18T18:24:18,749 INFO [master/961a81da9827:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-18T18:24:18,806 INFO [RS:1;961a81da9827:36379 {}] regionserver.HRegionServer(746): ClusterId : d961f11e-5c15-416e-a960-3926a0e53aca 2024-11-18T18:24:18,806 INFO [RS:0;961a81da9827:38709 {}] regionserver.HRegionServer(746): ClusterId : d961f11e-5c15-416e-a960-3926a0e53aca 2024-11-18T18:24:18,807 INFO [RS:2;961a81da9827:42895 {}] regionserver.HRegionServer(746): ClusterId : d961f11e-5c15-416e-a960-3926a0e53aca 2024-11-18T18:24:18,811 DEBUG [RS:1;961a81da9827:36379 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-18T18:24:18,811 DEBUG [RS:2;961a81da9827:42895 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-18T18:24:18,811 DEBUG [RS:0;961a81da9827:38709 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-18T18:24:18,817 DEBUG [RS:1;961a81da9827:36379 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-18T18:24:18,817 DEBUG [RS:0;961a81da9827:38709 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-18T18:24:18,817 DEBUG [RS:2;961a81da9827:42895 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-18T18:24:18,817 DEBUG [RS:1;961a81da9827:36379 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-18T18:24:18,817 DEBUG [RS:0;961a81da9827:38709 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-18T18:24:18,817 DEBUG [RS:2;961a81da9827:42895 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-18T18:24:18,822 DEBUG [RS:0;961a81da9827:38709 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-18T18:24:18,822 DEBUG [RS:1;961a81da9827:36379 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-18T18:24:18,822 DEBUG [RS:2;961a81da9827:42895 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-18T18:24:18,822 DEBUG [RS:0;961a81da9827:38709 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c33510a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=961a81da9827/172.17.0.2:0 2024-11-18T18:24:18,822 DEBUG [RS:2;961a81da9827:42895 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b4cb90d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=961a81da9827/172.17.0.2:0 2024-11-18T18:24:18,822 DEBUG [RS:1;961a81da9827:36379 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@512529fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=961a81da9827/172.17.0.2:0 2024-11-18T18:24:18,830 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-18T18:24:18,842 INFO [master/961a81da9827:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-18T18:24:18,844 DEBUG [RS:2;961a81da9827:42895 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;961a81da9827:42895 2024-11-18T18:24:18,845 DEBUG [RS:1;961a81da9827:36379 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;961a81da9827:36379 2024-11-18T18:24:18,845 DEBUG [RS:0;961a81da9827:38709 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;961a81da9827:38709 2024-11-18T18:24:18,848 INFO [RS:2;961a81da9827:42895 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-18T18:24:18,848 INFO [RS:1;961a81da9827:36379 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-18T18:24:18,848 INFO [RS:0;961a81da9827:38709 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-18T18:24:18,848 INFO [RS:2;961a81da9827:42895 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-18T18:24:18,848 INFO [RS:0;961a81da9827:38709 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-18T18:24:18,848 INFO [RS:1;961a81da9827:36379 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-18T18:24:18,848 DEBUG [RS:2;961a81da9827:42895 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-18T18:24:18,848 DEBUG [RS:1;961a81da9827:36379 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-18T18:24:18,848 DEBUG [RS:0;961a81da9827:38709 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-18T18:24:18,851 INFO [master/961a81da9827:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-18T18:24:18,852 INFO [RS:1;961a81da9827:36379 {}] regionserver.HRegionServer(2659): reportForDuty to master=961a81da9827,33089,1731954257001 with port=36379, startcode=1731954257826 2024-11-18T18:24:18,852 INFO [RS:2;961a81da9827:42895 {}] regionserver.HRegionServer(2659): reportForDuty to master=961a81da9827,33089,1731954257001 with port=42895, startcode=1731954257867 2024-11-18T18:24:18,852 INFO [RS:0;961a81da9827:38709 {}] regionserver.HRegionServer(2659): reportForDuty to master=961a81da9827,33089,1731954257001 with port=38709, startcode=1731954257723 2024-11-18T18:24:18,858 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 961a81da9827,33089,1731954257001 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-18T18:24:18,866 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/961a81da9827:0, corePoolSize=5, maxPoolSize=5 2024-11-18T18:24:18,866 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/961a81da9827:0, corePoolSize=5, maxPoolSize=5 2024-11-18T18:24:18,866 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/961a81da9827:0, corePoolSize=5, maxPoolSize=5 2024-11-18T18:24:18,866 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/961a81da9827:0, corePoolSize=5, maxPoolSize=5 2024-11-18T18:24:18,867 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/961a81da9827:0, corePoolSize=10, maxPoolSize=10 2024-11-18T18:24:18,867 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/961a81da9827:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:24:18,867 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/961a81da9827:0, corePoolSize=2, maxPoolSize=2 2024-11-18T18:24:18,867 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/961a81da9827:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:24:18,868 INFO [master/961a81da9827:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731954288868 2024-11-18T18:24:18,869 DEBUG [RS:2;961a81da9827:42895 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-18T18:24:18,869 DEBUG [RS:1;961a81da9827:36379 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-18T18:24:18,869 DEBUG [RS:0;961a81da9827:38709 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-18T18:24:18,870 INFO [master/961a81da9827:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-18T18:24:18,871 INFO [master/961a81da9827:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-18T18:24:18,876 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T18:24:18,876 INFO [master/961a81da9827:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-18T18:24:18,876 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-18T18:24:18,876 INFO [master/961a81da9827:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-18T18:24:18,877 INFO [master/961a81da9827:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-18T18:24:18,877 INFO [master/961a81da9827:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-18T18:24:18,878 INFO [master/961a81da9827:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-18T18:24:18,884 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:18,884 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-18T18:24:18,890 INFO [master/961a81da9827:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-18T18:24:18,891 INFO [master/961a81da9827:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-18T18:24:18,891 INFO [master/961a81da9827:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-18T18:24:18,894 INFO [master/961a81da9827:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-18T18:24:18,894 INFO [master/961a81da9827:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-18T18:24:18,897 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/961a81da9827:0:becomeActiveMaster-HFileCleaner.large.0-1731954258896,5,FailOnTimeoutGroup] 2024-11-18T18:24:18,903 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/961a81da9827:0:becomeActiveMaster-HFileCleaner.small.0-1731954258897,5,FailOnTimeoutGroup] 2024-11-18T18:24:18,903 INFO [master/961a81da9827:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-18T18:24:18,904 INFO [master/961a81da9827:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-18T18:24:18,906 INFO [master/961a81da9827:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-18T18:24:18,907 INFO [master/961a81da9827:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-18T18:24:18,953 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48665, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-18T18:24:18,956 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38043, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-18T18:24:18,958 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38743, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-18T18:24:18,964 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33089 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 961a81da9827,36379,1731954257826 2024-11-18T18:24:18,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741831_1007 (size=1321) 2024-11-18T18:24:18,967 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33089 {}] master.ServerManager(517): Registering regionserver=961a81da9827,36379,1731954257826 2024-11-18T18:24:18,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741831_1007 (size=1321) 2024-11-18T18:24:18,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741831_1007 (size=1321) 2024-11-18T18:24:18,972 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-18T18:24:18,972 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93 2024-11-18T18:24:18,980 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33089 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 961a81da9827,42895,1731954257867 2024-11-18T18:24:18,981 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33089 {}] master.ServerManager(517): Registering regionserver=961a81da9827,42895,1731954257867 2024-11-18T18:24:18,985 DEBUG [RS:1;961a81da9827:36379 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93 2024-11-18T18:24:18,985 DEBUG [RS:1;961a81da9827:36379 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43935 2024-11-18T18:24:18,985 DEBUG [RS:1;961a81da9827:36379 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-18T18:24:18,987 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33089 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 961a81da9827,38709,1731954257723 2024-11-18T18:24:18,987 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33089 {}] master.ServerManager(517): Registering regionserver=961a81da9827,38709,1731954257723 2024-11-18T18:24:18,989 DEBUG [RS:2;961a81da9827:42895 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93 2024-11-18T18:24:18,989 DEBUG [RS:2;961a81da9827:42895 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43935 2024-11-18T18:24:18,989 DEBUG [RS:2;961a81da9827:42895 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-18T18:24:18,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33089-0x10107179d9f0000, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T18:24:18,992 DEBUG [RS:0;961a81da9827:38709 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93 2024-11-18T18:24:18,992 DEBUG [RS:0;961a81da9827:38709 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43935 2024-11-18T18:24:18,992 DEBUG [RS:0;961a81da9827:38709 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-18T18:24:18,995 DEBUG [RS:1;961a81da9827:36379 {}] zookeeper.ZKUtil(111): regionserver:36379-0x10107179d9f0002, quorum=127.0.0.1:59292, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/961a81da9827,36379,1731954257826 2024-11-18T18:24:18,995 WARN [RS:1;961a81da9827:36379 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T18:24:18,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741832_1008 (size=32) 2024-11-18T18:24:18,996 INFO [RS:1;961a81da9827:36379 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T18:24:18,996 DEBUG [RS:1;961a81da9827:36379 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/WALs/961a81da9827,36379,1731954257826 2024-11-18T18:24:18,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741832_1008 (size=32) 2024-11-18T18:24:18,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33089-0x10107179d9f0000, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T18:24:18,998 DEBUG [RS:2;961a81da9827:42895 {}] zookeeper.ZKUtil(111): regionserver:42895-0x10107179d9f0003, quorum=127.0.0.1:59292, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/961a81da9827,42895,1731954257867 2024-11-18T18:24:18,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741832_1008 (size=32) 2024-11-18T18:24:18,998 WARN [RS:2;961a81da9827:42895 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T18:24:18,998 INFO [RS:2;961a81da9827:42895 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T18:24:18,998 DEBUG [RS:2;961a81da9827:42895 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/WALs/961a81da9827,42895,1731954257867 2024-11-18T18:24:18,999 DEBUG [RS:0;961a81da9827:38709 {}] zookeeper.ZKUtil(111): regionserver:38709-0x10107179d9f0001, quorum=127.0.0.1:59292, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/961a81da9827,38709,1731954257723 2024-11-18T18:24:18,999 WARN [RS:0;961a81da9827:38709 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T18:24:18,999 INFO [RS:0;961a81da9827:38709 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T18:24:18,999 DEBUG [RS:0;961a81da9827:38709 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/WALs/961a81da9827,38709,1731954257723 2024-11-18T18:24:19,000 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:19,001 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [961a81da9827,36379,1731954257826] 2024-11-18T18:24:19,002 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [961a81da9827,42895,1731954257867] 2024-11-18T18:24:19,002 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [961a81da9827,38709,1731954257723] 2024-11-18T18:24:19,006 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-18T18:24:19,009 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-18T18:24:19,009 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:19,010 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T18:24:19,011 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-18T18:24:19,018 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-18T18:24:19,019 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:19,020 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T18:24:19,020 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-18T18:24:19,038 INFO [RS:1;961a81da9827:36379 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-18T18:24:19,039 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-18T18:24:19,039 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:19,043 INFO [RS:0;961a81da9827:38709 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-18T18:24:19,046 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T18:24:19,046 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-18T18:24:19,048 INFO [RS:2;961a81da9827:42895 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-18T18:24:19,054 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-18T18:24:19,055 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:19,056 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T18:24:19,056 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-18T18:24:19,059 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/hbase/meta/1588230740 2024-11-18T18:24:19,060 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/hbase/meta/1588230740 2024-11-18T18:24:19,066 INFO [RS:1;961a81da9827:36379 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-18T18:24:19,069 INFO [RS:0;961a81da9827:38709 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-18T18:24:19,074 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-18T18:24:19,074 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-18T18:24:19,075 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T18:24:19,075 INFO [RS:1;961a81da9827:36379 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-18T18:24:19,075 INFO [RS:1;961a81da9827:36379 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T18:24:19,077 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-18T18:24:19,079 INFO [RS:2;961a81da9827:42895 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-18T18:24:19,090 INFO [RS:0;961a81da9827:38709 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-18T18:24:19,090 INFO [RS:0;961a81da9827:38709 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T18:24:19,091 INFO [RS:1;961a81da9827:36379 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-18T18:24:19,093 INFO [RS:2;961a81da9827:42895 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-18T18:24:19,093 INFO [RS:2;961a81da9827:42895 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T18:24:19,094 INFO [RS:2;961a81da9827:42895 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-18T18:24:19,098 INFO [RS:2;961a81da9827:42895 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-18T18:24:19,099 INFO [RS:1;961a81da9827:36379 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-18T18:24:19,101 INFO [RS:2;961a81da9827:42895 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-18T18:24:19,101 INFO [RS:0;961a81da9827:38709 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-18T18:24:19,101 DEBUG [RS:2;961a81da9827:42895 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/961a81da9827:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:24:19,101 DEBUG [RS:2;961a81da9827:42895 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/961a81da9827:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:24:19,101 INFO [RS:1;961a81da9827:36379 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-18T18:24:19,101 DEBUG [RS:2;961a81da9827:42895 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:24:19,102 DEBUG [RS:1;961a81da9827:36379 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/961a81da9827:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:24:19,102 DEBUG [RS:2;961a81da9827:42895 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/961a81da9827:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:24:19,102 DEBUG [RS:1;961a81da9827:36379 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/961a81da9827:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:24:19,104 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T18:24:19,104 DEBUG [RS:1;961a81da9827:36379 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:24:19,104 DEBUG [RS:2;961a81da9827:42895 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/961a81da9827:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:24:19,105 DEBUG [RS:1;961a81da9827:36379 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/961a81da9827:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:24:19,105 DEBUG [RS:2;961a81da9827:42895 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0, corePoolSize=2, maxPoolSize=2 2024-11-18T18:24:19,105 DEBUG [RS:1;961a81da9827:36379 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/961a81da9827:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:24:19,105 DEBUG [RS:2;961a81da9827:42895 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/961a81da9827:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:24:19,105 DEBUG [RS:1;961a81da9827:36379 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0, corePoolSize=2, maxPoolSize=2 2024-11-18T18:24:19,105 DEBUG [RS:2;961a81da9827:42895 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/961a81da9827:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:24:19,105 DEBUG [RS:1;961a81da9827:36379 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/961a81da9827:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:24:19,105 INFO [RS:0;961a81da9827:38709 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-18T18:24:19,105 DEBUG [RS:2;961a81da9827:42895 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/961a81da9827:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:24:19,105 DEBUG [RS:1;961a81da9827:36379 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/961a81da9827:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:24:19,105 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59117080, jitterRate=-0.1190868616104126}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T18:24:19,106 INFO [RS:0;961a81da9827:38709 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-18T18:24:19,106 DEBUG [RS:2;961a81da9827:42895 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/961a81da9827:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:24:19,106 DEBUG [RS:1;961a81da9827:36379 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/961a81da9827:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:24:19,106 DEBUG [RS:2;961a81da9827:42895 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/961a81da9827:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:24:19,106 DEBUG [RS:1;961a81da9827:36379 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/961a81da9827:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:24:19,106 DEBUG [RS:0;961a81da9827:38709 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/961a81da9827:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:24:19,106 DEBUG [RS:2;961a81da9827:42895 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/961a81da9827:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:24:19,106 DEBUG [RS:1;961a81da9827:36379 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/961a81da9827:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:24:19,106 DEBUG [RS:0;961a81da9827:38709 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/961a81da9827:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:24:19,106 DEBUG [RS:2;961a81da9827:42895 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/961a81da9827:0, corePoolSize=3, maxPoolSize=3 2024-11-18T18:24:19,106 DEBUG [RS:1;961a81da9827:36379 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/961a81da9827:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:24:19,106 DEBUG [RS:0;961a81da9827:38709 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:24:19,106 DEBUG [RS:2;961a81da9827:42895 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/961a81da9827:0, corePoolSize=3, maxPoolSize=3 2024-11-18T18:24:19,106 DEBUG [RS:1;961a81da9827:36379 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/961a81da9827:0, corePoolSize=3, maxPoolSize=3 2024-11-18T18:24:19,106 DEBUG [RS:0;961a81da9827:38709 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/961a81da9827:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:24:19,107 DEBUG [RS:1;961a81da9827:36379 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/961a81da9827:0, corePoolSize=3, maxPoolSize=3 2024-11-18T18:24:19,107 DEBUG [RS:0;961a81da9827:38709 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/961a81da9827:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:24:19,107 DEBUG [RS:0;961a81da9827:38709 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0, corePoolSize=2, maxPoolSize=2 2024-11-18T18:24:19,107 DEBUG [RS:0;961a81da9827:38709 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/961a81da9827:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:24:19,107 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731954259000Initializing all the Stores at 1731954259002 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731954259003 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731954259006 (+3 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954259006Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731954259006Cleaning up temporary data from old regions at 1731954259074 (+68 ms)Region opened successfully at 1731954259107 (+33 ms) 2024-11-18T18:24:19,107 DEBUG [RS:0;961a81da9827:38709 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/961a81da9827:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:24:19,107 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-18T18:24:19,107 DEBUG [RS:0;961a81da9827:38709 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/961a81da9827:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:24:19,107 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-18T18:24:19,108 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-18T18:24:19,108 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-18T18:24:19,108 DEBUG [RS:0;961a81da9827:38709 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/961a81da9827:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:24:19,108 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-18T18:24:19,108 DEBUG [RS:0;961a81da9827:38709 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/961a81da9827:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:24:19,108 DEBUG [RS:0;961a81da9827:38709 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/961a81da9827:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:24:19,108 DEBUG [RS:0;961a81da9827:38709 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/961a81da9827:0, corePoolSize=3, maxPoolSize=3 2024-11-18T18:24:19,108 DEBUG [RS:0;961a81da9827:38709 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/961a81da9827:0, corePoolSize=3, maxPoolSize=3 2024-11-18T18:24:19,109 INFO [RS:2;961a81da9827:42895 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T18:24:19,109 INFO [RS:2;961a81da9827:42895 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T18:24:19,109 INFO [RS:2;961a81da9827:42895 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T18:24:19,109 INFO [RS:2;961a81da9827:42895 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-18T18:24:19,109 INFO [RS:2;961a81da9827:42895 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-18T18:24:19,110 INFO [RS:2;961a81da9827:42895 {}] hbase.ChoreService(168): Chore ScheduledChore name=961a81da9827,42895,1731954257867-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T18:24:19,110 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-18T18:24:19,110 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731954259107Disabling compacts and flushes for region at 1731954259107Disabling writes for close at 1731954259108 (+1 ms)Writing region close event to WAL at 1731954259109 (+1 ms)Closed at 1731954259110 (+1 ms) 2024-11-18T18:24:19,112 INFO [RS:1;961a81da9827:36379 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T18:24:19,112 INFO [RS:1;961a81da9827:36379 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T18:24:19,112 INFO [RS:1;961a81da9827:36379 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T18:24:19,112 INFO [RS:1;961a81da9827:36379 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-18T18:24:19,112 INFO [RS:1;961a81da9827:36379 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-18T18:24:19,112 INFO [RS:1;961a81da9827:36379 {}] hbase.ChoreService(168): Chore ScheduledChore name=961a81da9827,36379,1731954257826-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T18:24:19,114 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T18:24:19,115 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-18T18:24:19,122 INFO [RS:0;961a81da9827:38709 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T18:24:19,122 INFO [RS:0;961a81da9827:38709 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T18:24:19,122 INFO [RS:0;961a81da9827:38709 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T18:24:19,122 INFO [RS:0;961a81da9827:38709 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-18T18:24:19,122 INFO [RS:0;961a81da9827:38709 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-18T18:24:19,122 INFO [RS:0;961a81da9827:38709 {}] hbase.ChoreService(168): Chore ScheduledChore name=961a81da9827,38709,1731954257723-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T18:24:19,123 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-18T18:24:19,139 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T18:24:19,143 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-18T18:24:19,150 INFO [RS:2;961a81da9827:42895 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-18T18:24:19,152 INFO [RS:2;961a81da9827:42895 {}] hbase.ChoreService(168): Chore ScheduledChore name=961a81da9827,42895,1731954257867-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T18:24:19,153 INFO [RS:2;961a81da9827:42895 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T18:24:19,153 INFO [RS:2;961a81da9827:42895 {}] regionserver.Replication(171): 961a81da9827,42895,1731954257867 started 2024-11-18T18:24:19,156 INFO [RS:1;961a81da9827:36379 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-18T18:24:19,157 INFO [RS:1;961a81da9827:36379 {}] hbase.ChoreService(168): Chore ScheduledChore name=961a81da9827,36379,1731954257826-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T18:24:19,157 INFO [RS:1;961a81da9827:36379 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T18:24:19,157 INFO [RS:1;961a81da9827:36379 {}] regionserver.Replication(171): 961a81da9827,36379,1731954257826 started 2024-11-18T18:24:19,173 INFO [RS:0;961a81da9827:38709 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-18T18:24:19,173 INFO [RS:0;961a81da9827:38709 {}] hbase.ChoreService(168): Chore ScheduledChore name=961a81da9827,38709,1731954257723-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T18:24:19,174 INFO [RS:0;961a81da9827:38709 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T18:24:19,174 INFO [RS:0;961a81da9827:38709 {}] regionserver.Replication(171): 961a81da9827,38709,1731954257723 started 2024-11-18T18:24:19,174 INFO [RS:1;961a81da9827:36379 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T18:24:19,174 INFO [RS:2;961a81da9827:42895 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T18:24:19,175 INFO [RS:2;961a81da9827:42895 {}] regionserver.HRegionServer(1482): Serving as 961a81da9827,42895,1731954257867, RpcServer on 961a81da9827/172.17.0.2:42895, sessionid=0x10107179d9f0003 2024-11-18T18:24:19,175 INFO [RS:1;961a81da9827:36379 {}] regionserver.HRegionServer(1482): Serving as 961a81da9827,36379,1731954257826, RpcServer on 961a81da9827/172.17.0.2:36379, sessionid=0x10107179d9f0002 2024-11-18T18:24:19,176 DEBUG [RS:2;961a81da9827:42895 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-18T18:24:19,176 DEBUG [RS:1;961a81da9827:36379 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-18T18:24:19,176 DEBUG [RS:2;961a81da9827:42895 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 961a81da9827,42895,1731954257867 2024-11-18T18:24:19,176 DEBUG [RS:1;961a81da9827:36379 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 961a81da9827,36379,1731954257826 2024-11-18T18:24:19,176 DEBUG [RS:2;961a81da9827:42895 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '961a81da9827,42895,1731954257867' 2024-11-18T18:24:19,176 DEBUG [RS:1;961a81da9827:36379 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '961a81da9827,36379,1731954257826' 2024-11-18T18:24:19,176 DEBUG [RS:2;961a81da9827:42895 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-18T18:24:19,176 DEBUG [RS:1;961a81da9827:36379 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-18T18:24:19,177 DEBUG [RS:2;961a81da9827:42895 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-18T18:24:19,177 DEBUG [RS:1;961a81da9827:36379 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-18T18:24:19,178 DEBUG [RS:1;961a81da9827:36379 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-18T18:24:19,178 DEBUG [RS:2;961a81da9827:42895 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-18T18:24:19,178 DEBUG [RS:1;961a81da9827:36379 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-18T18:24:19,178 DEBUG [RS:2;961a81da9827:42895 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-18T18:24:19,178 DEBUG [RS:1;961a81da9827:36379 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 961a81da9827,36379,1731954257826 2024-11-18T18:24:19,178 DEBUG [RS:2;961a81da9827:42895 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 961a81da9827,42895,1731954257867 2024-11-18T18:24:19,179 DEBUG [RS:1;961a81da9827:36379 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '961a81da9827,36379,1731954257826' 2024-11-18T18:24:19,179 DEBUG [RS:2;961a81da9827:42895 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '961a81da9827,42895,1731954257867' 2024-11-18T18:24:19,179 DEBUG [RS:1;961a81da9827:36379 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-18T18:24:19,179 DEBUG [RS:2;961a81da9827:42895 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-18T18:24:19,179 DEBUG [RS:2;961a81da9827:42895 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-18T18:24:19,179 DEBUG [RS:1;961a81da9827:36379 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-18T18:24:19,180 DEBUG [RS:2;961a81da9827:42895 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-18T18:24:19,180 INFO [RS:2;961a81da9827:42895 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-18T18:24:19,180 INFO [RS:2;961a81da9827:42895 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-18T18:24:19,182 DEBUG [RS:1;961a81da9827:36379 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-18T18:24:19,182 INFO [RS:1;961a81da9827:36379 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-18T18:24:19,182 INFO [RS:1;961a81da9827:36379 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-18T18:24:19,198 INFO [RS:0;961a81da9827:38709 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T18:24:19,198 INFO [RS:0;961a81da9827:38709 {}] regionserver.HRegionServer(1482): Serving as 961a81da9827,38709,1731954257723, RpcServer on 961a81da9827/172.17.0.2:38709, sessionid=0x10107179d9f0001 2024-11-18T18:24:19,199 DEBUG [RS:0;961a81da9827:38709 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-18T18:24:19,199 DEBUG [RS:0;961a81da9827:38709 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 961a81da9827,38709,1731954257723 2024-11-18T18:24:19,199 DEBUG [RS:0;961a81da9827:38709 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '961a81da9827,38709,1731954257723' 2024-11-18T18:24:19,199 DEBUG [RS:0;961a81da9827:38709 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-18T18:24:19,200 DEBUG [RS:0;961a81da9827:38709 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-18T18:24:19,201 DEBUG [RS:0;961a81da9827:38709 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-18T18:24:19,201 DEBUG [RS:0;961a81da9827:38709 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-18T18:24:19,201 DEBUG [RS:0;961a81da9827:38709 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 961a81da9827,38709,1731954257723 2024-11-18T18:24:19,201 DEBUG [RS:0;961a81da9827:38709 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '961a81da9827,38709,1731954257723' 2024-11-18T18:24:19,201 DEBUG [RS:0;961a81da9827:38709 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-18T18:24:19,201 DEBUG [RS:0;961a81da9827:38709 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-18T18:24:19,202 DEBUG [RS:0;961a81da9827:38709 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-18T18:24:19,202 INFO [RS:0;961a81da9827:38709 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-18T18:24:19,202 INFO [RS:0;961a81da9827:38709 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-18T18:24:19,286 INFO [RS:1;961a81da9827:36379 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T18:24:19,286 INFO [RS:2;961a81da9827:42895 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T18:24:19,290 INFO [RS:2;961a81da9827:42895 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=961a81da9827%2C42895%2C1731954257867, suffix=, logDir=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/WALs/961a81da9827,42895,1731954257867, archiveDir=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/oldWALs, maxLogs=32 2024-11-18T18:24:19,290 INFO [RS:1;961a81da9827:36379 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=961a81da9827%2C36379%2C1731954257826, suffix=, logDir=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/WALs/961a81da9827,36379,1731954257826, archiveDir=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/oldWALs, maxLogs=32 2024-11-18T18:24:19,294 WARN [961a81da9827:33089 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:24:19,303 INFO [RS:0;961a81da9827:38709 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T18:24:19,306 INFO [RS:0;961a81da9827:38709 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=961a81da9827%2C38709%2C1731954257723, suffix=, logDir=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/WALs/961a81da9827,38709,1731954257723, archiveDir=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/oldWALs, maxLogs=32 2024-11-18T18:24:19,310 DEBUG [RS:1;961a81da9827:36379 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/WALs/961a81da9827,36379,1731954257826/961a81da9827%2C36379%2C1731954257826.1731954259295, exclude list is [], retry=0 2024-11-18T18:24:19,313 DEBUG [RS:2;961a81da9827:42895 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/WALs/961a81da9827,42895,1731954257867/961a81da9827%2C42895%2C1731954257867.1731954259295, exclude list is [], retry=0 2024-11-18T18:24:19,316 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44191,DS-39ff43bf-c18b-4b86-b37c-eca99a433383,DISK] 2024-11-18T18:24:19,316 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41713,DS-c75b1747-362b-431f-b47d-543a4f882cdb,DISK] 2024-11-18T18:24:19,316 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34047,DS-1029886b-6b51-4ae4-81a3-156afc7de901,DISK] 2024-11-18T18:24:19,318 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34047,DS-1029886b-6b51-4ae4-81a3-156afc7de901,DISK] 2024-11-18T18:24:19,318 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44191,DS-39ff43bf-c18b-4b86-b37c-eca99a433383,DISK] 2024-11-18T18:24:19,318 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41713,DS-c75b1747-362b-431f-b47d-543a4f882cdb,DISK] 2024-11-18T18:24:19,325 INFO [RS:1;961a81da9827:36379 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/WALs/961a81da9827,36379,1731954257826/961a81da9827%2C36379%2C1731954257826.1731954259295 2024-11-18T18:24:19,325 INFO [RS:2;961a81da9827:42895 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/WALs/961a81da9827,42895,1731954257867/961a81da9827%2C42895%2C1731954257867.1731954259295 2024-11-18T18:24:19,326 DEBUG [RS:1;961a81da9827:36379 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34389:34389),(127.0.0.1/127.0.0.1:41643:41643),(127.0.0.1/127.0.0.1:38097:38097)] 2024-11-18T18:24:19,326 DEBUG [RS:0;961a81da9827:38709 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/WALs/961a81da9827,38709,1731954257723/961a81da9827%2C38709%2C1731954257723.1731954259308, exclude list is [], retry=0 2024-11-18T18:24:19,326 DEBUG [RS:2;961a81da9827:42895 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38097:38097),(127.0.0.1/127.0.0.1:34389:34389),(127.0.0.1/127.0.0.1:41643:41643)] 2024-11-18T18:24:19,333 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41713,DS-c75b1747-362b-431f-b47d-543a4f882cdb,DISK] 2024-11-18T18:24:19,333 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34047,DS-1029886b-6b51-4ae4-81a3-156afc7de901,DISK] 2024-11-18T18:24:19,334 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44191,DS-39ff43bf-c18b-4b86-b37c-eca99a433383,DISK] 2024-11-18T18:24:19,341 INFO [RS:0;961a81da9827:38709 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/WALs/961a81da9827,38709,1731954257723/961a81da9827%2C38709%2C1731954257723.1731954259308 2024-11-18T18:24:19,342 DEBUG [RS:0;961a81da9827:38709 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41643:41643),(127.0.0.1/127.0.0.1:34389:34389),(127.0.0.1/127.0.0.1:38097:38097)] 2024-11-18T18:24:19,547 DEBUG [961a81da9827:33089 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-18T18:24:19,555 DEBUG [961a81da9827:33089 {}] balancer.BalancerClusterState(204): Hosts are {961a81da9827=0} racks are {/default-rack=0} 2024-11-18T18:24:19,563 DEBUG [961a81da9827:33089 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-18T18:24:19,563 DEBUG [961a81da9827:33089 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-18T18:24:19,563 DEBUG [961a81da9827:33089 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-18T18:24:19,563 DEBUG [961a81da9827:33089 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-18T18:24:19,563 DEBUG [961a81da9827:33089 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-18T18:24:19,563 DEBUG [961a81da9827:33089 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-18T18:24:19,563 INFO [961a81da9827:33089 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-18T18:24:19,563 INFO [961a81da9827:33089 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-18T18:24:19,563 INFO [961a81da9827:33089 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-18T18:24:19,563 DEBUG [961a81da9827:33089 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-18T18:24:19,570 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=961a81da9827,36379,1731954257826 2024-11-18T18:24:19,578 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 961a81da9827,36379,1731954257826, state=OPENING 2024-11-18T18:24:19,583 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-18T18:24:19,585 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33089-0x10107179d9f0000, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:24:19,585 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38709-0x10107179d9f0001, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:24:19,585 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42895-0x10107179d9f0003, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:24:19,585 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36379-0x10107179d9f0002, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:24:19,586 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:24:19,586 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:24:19,586 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:24:19,586 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:24:19,588 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T18:24:19,589 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=961a81da9827,36379,1731954257826}] 2024-11-18T18:24:19,766 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-18T18:24:19,769 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58717, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-18T18:24:19,783 INFO [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-18T18:24:19,783 INFO [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T18:24:19,783 INFO [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-18T18:24:19,787 INFO [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=961a81da9827%2C36379%2C1731954257826.meta, suffix=.meta, logDir=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/WALs/961a81da9827,36379,1731954257826, archiveDir=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/oldWALs, maxLogs=32 2024-11-18T18:24:19,804 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/WALs/961a81da9827,36379,1731954257826/961a81da9827%2C36379%2C1731954257826.meta.1731954259788.meta, exclude list is [], retry=0 2024-11-18T18:24:19,807 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41713,DS-c75b1747-362b-431f-b47d-543a4f882cdb,DISK] 2024-11-18T18:24:19,807 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34047,DS-1029886b-6b51-4ae4-81a3-156afc7de901,DISK] 2024-11-18T18:24:19,808 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44191,DS-39ff43bf-c18b-4b86-b37c-eca99a433383,DISK] 2024-11-18T18:24:19,812 INFO [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/WALs/961a81da9827,36379,1731954257826/961a81da9827%2C36379%2C1731954257826.meta.1731954259788.meta 2024-11-18T18:24:19,812 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38097:38097),(127.0.0.1/127.0.0.1:41643:41643),(127.0.0.1/127.0.0.1:34389:34389)] 2024-11-18T18:24:19,813 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-18T18:24:19,814 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-18T18:24:19,817 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-18T18:24:19,822 INFO [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-18T18:24:19,826 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-18T18:24:19,827 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:19,827 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-18T18:24:19,827 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-18T18:24:19,830 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-18T18:24:19,832 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-18T18:24:19,832 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:19,833 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T18:24:19,833 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-18T18:24:19,835 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-18T18:24:19,835 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:19,836 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T18:24:19,836 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-18T18:24:19,837 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-18T18:24:19,837 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:19,838 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T18:24:19,838 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-18T18:24:19,839 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-18T18:24:19,839 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:19,840 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T18:24:19,841 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-18T18:24:19,842 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/hbase/meta/1588230740 2024-11-18T18:24:19,845 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/hbase/meta/1588230740 2024-11-18T18:24:19,848 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-18T18:24:19,848 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-18T18:24:19,849 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T18:24:19,852 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-18T18:24:19,855 INFO [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70830080, jitterRate=0.055450439453125}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T18:24:19,855 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-18T18:24:19,857 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731954259827Writing region info on filesystem at 1731954259828 (+1 ms)Initializing all the Stores at 1731954259830 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731954259830Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731954259830Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954259830Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731954259830Cleaning up temporary data from old regions at 1731954259848 (+18 ms)Running coprocessor post-open hooks at 1731954259855 (+7 ms)Region opened successfully at 1731954259857 (+2 ms) 2024-11-18T18:24:19,867 INFO [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731954259756 2024-11-18T18:24:19,881 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-18T18:24:19,881 INFO [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-18T18:24:19,883 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=961a81da9827,36379,1731954257826 2024-11-18T18:24:19,886 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 961a81da9827,36379,1731954257826, state=OPEN 2024-11-18T18:24:19,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33089-0x10107179d9f0000, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T18:24:19,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42895-0x10107179d9f0003, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T18:24:19,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36379-0x10107179d9f0002, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T18:24:19,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38709-0x10107179d9f0001, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T18:24:19,889 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:24:19,889 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:24:19,889 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:24:19,890 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:24:19,890 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=961a81da9827,36379,1731954257826 2024-11-18T18:24:19,896 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-18T18:24:19,896 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=961a81da9827,36379,1731954257826 in 301 msec 2024-11-18T18:24:19,905 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-18T18:24:19,905 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 775 msec 2024-11-18T18:24:19,907 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T18:24:19,907 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-18T18:24:19,928 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T18:24:19,930 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1] 2024-11-18T18:24:19,953 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T18:24:19,956 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49709, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T18:24:19,984 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2020 sec 2024-11-18T18:24:19,984 INFO [master/961a81da9827:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731954259984, completionTime=-1 2024-11-18T18:24:19,988 INFO [master/961a81da9827:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-18T18:24:19,988 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-18T18:24:20,021 INFO [master/961a81da9827:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-18T18:24:20,021 INFO [master/961a81da9827:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731954320021 2024-11-18T18:24:20,021 INFO [master/961a81da9827:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731954380021 2024-11-18T18:24:20,021 INFO [master/961a81da9827:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 33 msec 2024-11-18T18:24:20,023 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-18T18:24:20,031 INFO [master/961a81da9827:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=961a81da9827,33089,1731954257001-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T18:24:20,031 INFO [master/961a81da9827:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=961a81da9827,33089,1731954257001-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T18:24:20,031 INFO [master/961a81da9827:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=961a81da9827,33089,1731954257001-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T18:24:20,033 INFO [master/961a81da9827:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-961a81da9827:33089, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T18:24:20,033 INFO [master/961a81da9827:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-18T18:24:20,034 INFO [master/961a81da9827:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-18T18:24:20,041 DEBUG [master/961a81da9827:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-18T18:24:20,065 INFO [master/961a81da9827:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.098sec 2024-11-18T18:24:20,067 INFO [master/961a81da9827:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-18T18:24:20,069 INFO [master/961a81da9827:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-18T18:24:20,070 INFO [master/961a81da9827:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-18T18:24:20,071 INFO [master/961a81da9827:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-18T18:24:20,071 INFO [master/961a81da9827:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-18T18:24:20,072 INFO [master/961a81da9827:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=961a81da9827,33089,1731954257001-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T18:24:20,073 INFO [master/961a81da9827:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=961a81da9827,33089,1731954257001-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-18T18:24:20,078 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-18T18:24:20,079 INFO [master/961a81da9827:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-18T18:24:20,079 INFO [master/961a81da9827:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=961a81da9827,33089,1731954257001-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T18:24:20,114 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6baf23d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T18:24:20,119 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-18T18:24:20,119 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-18T18:24:20,123 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 961a81da9827,33089,-1 for getting cluster id 2024-11-18T18:24:20,125 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T18:24:20,135 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd961f11e-5c15-416e-a960-3926a0e53aca' 2024-11-18T18:24:20,138 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T18:24:20,138 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d961f11e-5c15-416e-a960-3926a0e53aca" 2024-11-18T18:24:20,139 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e9c5ec0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T18:24:20,139 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [961a81da9827,33089,-1] 2024-11-18T18:24:20,142 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T18:24:20,145 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T18:24:20,145 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50606, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T18:24:20,149 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63b8549a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T18:24:20,149 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T18:24:20,156 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1] 2024-11-18T18:24:20,157 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T18:24:20,160 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38530, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T18:24:20,214 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=961a81da9827,33089,1731954257001 2024-11-18T18:24:20,216 WARN [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false 2024-11-18T18:24:20,221 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T18:24:20,226 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 961a81da9827,33089,1731954257001 2024-11-18T18:24:20,228 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5496d838 2024-11-18T18:24:20,229 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T18:24:20,232 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50618, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T18:24:20,244 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33089 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T18:24:20,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33089 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-18T18:24:20,257 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T18:24:20,259 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33089 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true" procId is: 4 2024-11-18T18:24:20,260 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:20,264 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T18:24:20,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T18:24:20,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741837_1013 (size=1020) 2024-11-18T18:24:20,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741837_1013 (size=1020) 2024-11-18T18:24:20,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741837_1013 (size=1020) 2024-11-18T18:24:20,288 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 43138a6a12c2efea7093064622bb3cd8, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731954260238.43138a6a12c2efea7093064622bb3cd8.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93 2024-11-18T18:24:20,288 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8bb7ae0747eda2290799ca8fa83f9ede, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731954260238.8bb7ae0747eda2290799ca8fa83f9ede.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93 2024-11-18T18:24:20,288 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 22971deb0db38b7cccde5c3bc0868f22, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731954260238.22971deb0db38b7cccde5c3bc0868f22.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93 2024-11-18T18:24:20,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741838_1014 (size=110) 2024-11-18T18:24:20,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741838_1014 (size=110) 2024-11-18T18:24:20,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741838_1014 (size=110) 2024-11-18T18:24:20,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741840_1016 (size=110) 2024-11-18T18:24:20,328 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731954260238.22971deb0db38b7cccde5c3bc0868f22.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:20,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741839_1015 (size=107) 2024-11-18T18:24:20,328 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing 22971deb0db38b7cccde5c3bc0868f22, disabling compactions & flushes 2024-11-18T18:24:20,328 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731954260238.22971deb0db38b7cccde5c3bc0868f22. 2024-11-18T18:24:20,328 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731954260238.22971deb0db38b7cccde5c3bc0868f22. 2024-11-18T18:24:20,328 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731954260238.22971deb0db38b7cccde5c3bc0868f22. after waiting 0 ms 2024-11-18T18:24:20,328 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731954260238.22971deb0db38b7cccde5c3bc0868f22. 2024-11-18T18:24:20,328 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731954260238.22971deb0db38b7cccde5c3bc0868f22. 2024-11-18T18:24:20,328 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for 22971deb0db38b7cccde5c3bc0868f22: Waiting for close lock at 1731954260328Disabling compacts and flushes for region at 1731954260328Disabling writes for close at 1731954260328Writing region close event to WAL at 1731954260328Closed at 1731954260328 2024-11-18T18:24:20,329 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => bb099f0975ebdb618ef8e3b2c263bbca, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731954260238.bb099f0975ebdb618ef8e3b2c263bbca.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93 2024-11-18T18:24:20,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741840_1016 (size=110) 2024-11-18T18:24:20,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741840_1016 (size=110) 2024-11-18T18:24:20,331 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731954260238.43138a6a12c2efea7093064622bb3cd8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:20,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741839_1015 (size=107) 2024-11-18T18:24:20,331 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing 43138a6a12c2efea7093064622bb3cd8, disabling compactions & flushes 2024-11-18T18:24:20,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741839_1015 (size=107) 2024-11-18T18:24:20,331 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731954260238.43138a6a12c2efea7093064622bb3cd8. 2024-11-18T18:24:20,332 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731954260238.43138a6a12c2efea7093064622bb3cd8. 2024-11-18T18:24:20,332 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731954260238.43138a6a12c2efea7093064622bb3cd8. after waiting 0 ms 2024-11-18T18:24:20,332 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731954260238.43138a6a12c2efea7093064622bb3cd8. 2024-11-18T18:24:20,332 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731954260238.43138a6a12c2efea7093064622bb3cd8. 2024-11-18T18:24:20,332 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for 43138a6a12c2efea7093064622bb3cd8: Waiting for close lock at 1731954260331Disabling compacts and flushes for region at 1731954260331Disabling writes for close at 1731954260332 (+1 ms)Writing region close event to WAL at 1731954260332Closed at 1731954260332 2024-11-18T18:24:20,333 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => bcf8695060a326b7d24615ba6cb932b1, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731954260238.bcf8695060a326b7d24615ba6cb932b1.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93 2024-11-18T18:24:20,334 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731954260238.8bb7ae0747eda2290799ca8fa83f9ede.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:20,334 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 8bb7ae0747eda2290799ca8fa83f9ede, disabling compactions & flushes 2024-11-18T18:24:20,334 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731954260238.8bb7ae0747eda2290799ca8fa83f9ede. 2024-11-18T18:24:20,334 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731954260238.8bb7ae0747eda2290799ca8fa83f9ede. 2024-11-18T18:24:20,334 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731954260238.8bb7ae0747eda2290799ca8fa83f9ede. after waiting 0 ms 2024-11-18T18:24:20,334 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731954260238.8bb7ae0747eda2290799ca8fa83f9ede. 2024-11-18T18:24:20,334 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731954260238.8bb7ae0747eda2290799ca8fa83f9ede. 2024-11-18T18:24:20,334 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8bb7ae0747eda2290799ca8fa83f9ede: Waiting for close lock at 1731954260334Disabling compacts and flushes for region at 1731954260334Disabling writes for close at 1731954260334Writing region close event to WAL at 1731954260334Closed at 1731954260334 2024-11-18T18:24:20,335 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fb91ff28ef3c18f0dc0d3f26aac68eca, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731954260238.fb91ff28ef3c18f0dc0d3f26aac68eca.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93 2024-11-18T18:24:20,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741841_1017 (size=110) 2024-11-18T18:24:20,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741841_1017 (size=110) 2024-11-18T18:24:20,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741841_1017 (size=110) 2024-11-18T18:24:20,356 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731954260238.bb099f0975ebdb618ef8e3b2c263bbca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:20,357 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing bb099f0975ebdb618ef8e3b2c263bbca, disabling compactions & flushes 2024-11-18T18:24:20,357 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731954260238.bb099f0975ebdb618ef8e3b2c263bbca. 2024-11-18T18:24:20,357 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731954260238.bb099f0975ebdb618ef8e3b2c263bbca. 2024-11-18T18:24:20,357 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731954260238.bb099f0975ebdb618ef8e3b2c263bbca. after waiting 0 ms 2024-11-18T18:24:20,357 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731954260238.bb099f0975ebdb618ef8e3b2c263bbca. 2024-11-18T18:24:20,357 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731954260238.bb099f0975ebdb618ef8e3b2c263bbca. 2024-11-18T18:24:20,357 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for bb099f0975ebdb618ef8e3b2c263bbca: Waiting for close lock at 1731954260357Disabling compacts and flushes for region at 1731954260357Disabling writes for close at 1731954260357Writing region close event to WAL at 1731954260357Closed at 1731954260357 2024-11-18T18:24:20,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741842_1018 (size=110) 2024-11-18T18:24:20,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741842_1018 (size=110) 2024-11-18T18:24:20,358 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => c8b4dd2a6d8dd59085c547b55aac0de6, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731954260238.c8b4dd2a6d8dd59085c547b55aac0de6.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93 2024-11-18T18:24:20,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741842_1018 (size=110) 2024-11-18T18:24:20,360 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731954260238.bcf8695060a326b7d24615ba6cb932b1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:20,360 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing bcf8695060a326b7d24615ba6cb932b1, disabling compactions & flushes 2024-11-18T18:24:20,360 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731954260238.bcf8695060a326b7d24615ba6cb932b1. 2024-11-18T18:24:20,360 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731954260238.bcf8695060a326b7d24615ba6cb932b1. 2024-11-18T18:24:20,360 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731954260238.bcf8695060a326b7d24615ba6cb932b1. after waiting 0 ms 2024-11-18T18:24:20,360 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731954260238.bcf8695060a326b7d24615ba6cb932b1. 2024-11-18T18:24:20,360 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731954260238.bcf8695060a326b7d24615ba6cb932b1. 2024-11-18T18:24:20,360 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for bcf8695060a326b7d24615ba6cb932b1: Waiting for close lock at 1731954260360Disabling compacts and flushes for region at 1731954260360Disabling writes for close at 1731954260360Writing region close event to WAL at 1731954260360Closed at 1731954260360 2024-11-18T18:24:20,361 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 40a213dc6b2a0fd809c71612c437ab83, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731954260238.40a213dc6b2a0fd809c71612c437ab83.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93 2024-11-18T18:24:20,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741843_1019 (size=110) 2024-11-18T18:24:20,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741843_1019 (size=110) 2024-11-18T18:24:20,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741843_1019 (size=110) 2024-11-18T18:24:20,365 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731954260238.fb91ff28ef3c18f0dc0d3f26aac68eca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:20,365 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing fb91ff28ef3c18f0dc0d3f26aac68eca, disabling compactions & flushes 2024-11-18T18:24:20,365 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731954260238.fb91ff28ef3c18f0dc0d3f26aac68eca. 2024-11-18T18:24:20,365 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731954260238.fb91ff28ef3c18f0dc0d3f26aac68eca. 2024-11-18T18:24:20,365 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731954260238.fb91ff28ef3c18f0dc0d3f26aac68eca. after waiting 0 ms 2024-11-18T18:24:20,365 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731954260238.fb91ff28ef3c18f0dc0d3f26aac68eca. 2024-11-18T18:24:20,365 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731954260238.fb91ff28ef3c18f0dc0d3f26aac68eca. 2024-11-18T18:24:20,365 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for fb91ff28ef3c18f0dc0d3f26aac68eca: Waiting for close lock at 1731954260365Disabling compacts and flushes for region at 1731954260365Disabling writes for close at 1731954260365Writing region close event to WAL at 1731954260365Closed at 1731954260365 2024-11-18T18:24:20,366 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f39571a93e25091090ea0831d54936b5, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731954260238.f39571a93e25091090ea0831d54936b5.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93 2024-11-18T18:24:20,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741844_1020 (size=110) 2024-11-18T18:24:20,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741844_1020 (size=110) 2024-11-18T18:24:20,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741844_1020 (size=110) 2024-11-18T18:24:20,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T18:24:20,378 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731954260238.c8b4dd2a6d8dd59085c547b55aac0de6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:20,378 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing c8b4dd2a6d8dd59085c547b55aac0de6, disabling compactions & flushes 2024-11-18T18:24:20,378 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731954260238.c8b4dd2a6d8dd59085c547b55aac0de6. 2024-11-18T18:24:20,378 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731954260238.c8b4dd2a6d8dd59085c547b55aac0de6. 2024-11-18T18:24:20,378 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731954260238.c8b4dd2a6d8dd59085c547b55aac0de6. after waiting 0 ms 2024-11-18T18:24:20,378 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731954260238.c8b4dd2a6d8dd59085c547b55aac0de6. 2024-11-18T18:24:20,378 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731954260238.c8b4dd2a6d8dd59085c547b55aac0de6. 2024-11-18T18:24:20,379 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for c8b4dd2a6d8dd59085c547b55aac0de6: Waiting for close lock at 1731954260378Disabling compacts and flushes for region at 1731954260378Disabling writes for close at 1731954260378Writing region close event to WAL at 1731954260378Closed at 1731954260378 2024-11-18T18:24:20,379 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 97a938e093c14701871e85a1fbb6e07d, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731954260238.97a938e093c14701871e85a1fbb6e07d.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93 2024-11-18T18:24:20,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741845_1021 (size=110) 2024-11-18T18:24:20,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741845_1021 (size=110) 2024-11-18T18:24:20,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741845_1021 (size=110) 2024-11-18T18:24:20,385 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731954260238.40a213dc6b2a0fd809c71612c437ab83.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:20,385 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing 40a213dc6b2a0fd809c71612c437ab83, disabling compactions & flushes 2024-11-18T18:24:20,385 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731954260238.40a213dc6b2a0fd809c71612c437ab83. 2024-11-18T18:24:20,385 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731954260238.40a213dc6b2a0fd809c71612c437ab83. 2024-11-18T18:24:20,385 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731954260238.40a213dc6b2a0fd809c71612c437ab83. after waiting 0 ms 2024-11-18T18:24:20,385 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731954260238.40a213dc6b2a0fd809c71612c437ab83. 2024-11-18T18:24:20,385 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731954260238.40a213dc6b2a0fd809c71612c437ab83. 2024-11-18T18:24:20,385 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for 40a213dc6b2a0fd809c71612c437ab83: Waiting for close lock at 1731954260385Disabling compacts and flushes for region at 1731954260385Disabling writes for close at 1731954260385Writing region close event to WAL at 1731954260385Closed at 1731954260385 2024-11-18T18:24:20,386 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => c3e2f3fb6516fcaaef4d0a2c8b63842d, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731954260238.c3e2f3fb6516fcaaef4d0a2c8b63842d.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93 2024-11-18T18:24:20,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741846_1022 (size=110) 2024-11-18T18:24:20,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741846_1022 (size=110) 2024-11-18T18:24:20,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741846_1022 (size=110) 2024-11-18T18:24:20,388 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731954260238.f39571a93e25091090ea0831d54936b5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:20,389 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing f39571a93e25091090ea0831d54936b5, disabling compactions & flushes 2024-11-18T18:24:20,389 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731954260238.f39571a93e25091090ea0831d54936b5. 2024-11-18T18:24:20,389 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731954260238.f39571a93e25091090ea0831d54936b5. 2024-11-18T18:24:20,389 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731954260238.f39571a93e25091090ea0831d54936b5. after waiting 0 ms 2024-11-18T18:24:20,389 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731954260238.f39571a93e25091090ea0831d54936b5. 2024-11-18T18:24:20,389 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731954260238.f39571a93e25091090ea0831d54936b5. 2024-11-18T18:24:20,389 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for f39571a93e25091090ea0831d54936b5: Waiting for close lock at 1731954260389Disabling compacts and flushes for region at 1731954260389Disabling writes for close at 1731954260389Writing region close event to WAL at 1731954260389Closed at 1731954260389 2024-11-18T18:24:20,390 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 97377a91d35f8b242120f603c456c41e, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731954260238.97377a91d35f8b242120f603c456c41e.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93 2024-11-18T18:24:20,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741847_1023 (size=110) 2024-11-18T18:24:20,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741847_1023 (size=110) 2024-11-18T18:24:20,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741847_1023 (size=110) 2024-11-18T18:24:20,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741848_1024 (size=110) 2024-11-18T18:24:20,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741848_1024 (size=110) 2024-11-18T18:24:20,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741848_1024 (size=110) 2024-11-18T18:24:20,406 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731954260238.97a938e093c14701871e85a1fbb6e07d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:20,407 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing 97a938e093c14701871e85a1fbb6e07d, disabling compactions & flushes 2024-11-18T18:24:20,407 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731954260238.97a938e093c14701871e85a1fbb6e07d. 2024-11-18T18:24:20,407 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731954260238.97a938e093c14701871e85a1fbb6e07d. 2024-11-18T18:24:20,407 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731954260238.97a938e093c14701871e85a1fbb6e07d. after waiting 0 ms 2024-11-18T18:24:20,407 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731954260238.97a938e093c14701871e85a1fbb6e07d. 2024-11-18T18:24:20,407 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731954260238.97a938e093c14701871e85a1fbb6e07d. 2024-11-18T18:24:20,407 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for 97a938e093c14701871e85a1fbb6e07d: Waiting for close lock at 1731954260407Disabling compacts and flushes for region at 1731954260407Disabling writes for close at 1731954260407Writing region close event to WAL at 1731954260407Closed at 1731954260407 2024-11-18T18:24:20,407 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731954260238.c3e2f3fb6516fcaaef4d0a2c8b63842d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:20,408 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing c3e2f3fb6516fcaaef4d0a2c8b63842d, disabling compactions & flushes 2024-11-18T18:24:20,408 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => a244d0ab318eb4dd77bc5235828d9d51, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731954260238.a244d0ab318eb4dd77bc5235828d9d51.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93 2024-11-18T18:24:20,408 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731954260238.c3e2f3fb6516fcaaef4d0a2c8b63842d. 2024-11-18T18:24:20,408 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731954260238.c3e2f3fb6516fcaaef4d0a2c8b63842d. 2024-11-18T18:24:20,408 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731954260238.c3e2f3fb6516fcaaef4d0a2c8b63842d. after waiting 0 ms 2024-11-18T18:24:20,408 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731954260238.c3e2f3fb6516fcaaef4d0a2c8b63842d. 2024-11-18T18:24:20,408 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731954260238.c3e2f3fb6516fcaaef4d0a2c8b63842d. 2024-11-18T18:24:20,409 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for c3e2f3fb6516fcaaef4d0a2c8b63842d: Waiting for close lock at 1731954260408Disabling compacts and flushes for region at 1731954260408Disabling writes for close at 1731954260408Writing region close event to WAL at 1731954260408Closed at 1731954260408 2024-11-18T18:24:20,410 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 012c277c99c564dd90e8e5f6850b6ce0, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731954260238.012c277c99c564dd90e8e5f6850b6ce0.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93 2024-11-18T18:24:20,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741849_1025 (size=110) 2024-11-18T18:24:20,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741849_1025 (size=110) 2024-11-18T18:24:20,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741849_1025 (size=110) 2024-11-18T18:24:20,421 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731954260238.97377a91d35f8b242120f603c456c41e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:20,421 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 97377a91d35f8b242120f603c456c41e, disabling compactions & flushes 2024-11-18T18:24:20,422 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731954260238.97377a91d35f8b242120f603c456c41e. 2024-11-18T18:24:20,422 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731954260238.97377a91d35f8b242120f603c456c41e. 2024-11-18T18:24:20,422 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731954260238.97377a91d35f8b242120f603c456c41e. after waiting 0 ms 2024-11-18T18:24:20,422 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731954260238.97377a91d35f8b242120f603c456c41e. 2024-11-18T18:24:20,422 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731954260238.97377a91d35f8b242120f603c456c41e. 2024-11-18T18:24:20,422 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 97377a91d35f8b242120f603c456c41e: Waiting for close lock at 1731954260421Disabling compacts and flushes for region at 1731954260421Disabling writes for close at 1731954260422 (+1 ms)Writing region close event to WAL at 1731954260422Closed at 1731954260422 2024-11-18T18:24:20,423 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 140f5077d3b730b643b558ebe8a13abd, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731954260238.140f5077d3b730b643b558ebe8a13abd.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93 2024-11-18T18:24:20,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741850_1026 (size=110) 2024-11-18T18:24:20,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741850_1026 (size=110) 2024-11-18T18:24:20,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741851_1027 (size=110) 2024-11-18T18:24:20,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741851_1027 (size=110) 2024-11-18T18:24:20,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741851_1027 (size=110) 2024-11-18T18:24:20,432 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731954260238.012c277c99c564dd90e8e5f6850b6ce0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:20,432 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing 012c277c99c564dd90e8e5f6850b6ce0, disabling compactions & flushes 2024-11-18T18:24:20,432 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731954260238.012c277c99c564dd90e8e5f6850b6ce0. 2024-11-18T18:24:20,432 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731954260238.012c277c99c564dd90e8e5f6850b6ce0. 2024-11-18T18:24:20,432 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731954260238.012c277c99c564dd90e8e5f6850b6ce0. after waiting 0 ms 2024-11-18T18:24:20,433 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731954260238.012c277c99c564dd90e8e5f6850b6ce0. 2024-11-18T18:24:20,433 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731954260238.012c277c99c564dd90e8e5f6850b6ce0. 2024-11-18T18:24:20,433 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for 012c277c99c564dd90e8e5f6850b6ce0: Waiting for close lock at 1731954260432Disabling compacts and flushes for region at 1731954260432Disabling writes for close at 1731954260433 (+1 ms)Writing region close event to WAL at 1731954260433Closed at 1731954260433 2024-11-18T18:24:20,434 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 5e69ef2147d87d1b83dbcbb4705cbabe, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731954260238.5e69ef2147d87d1b83dbcbb4705cbabe.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93 2024-11-18T18:24:20,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741850_1026 (size=110) 2024-11-18T18:24:20,436 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731954260238.a244d0ab318eb4dd77bc5235828d9d51.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:20,436 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing a244d0ab318eb4dd77bc5235828d9d51, disabling compactions & flushes 2024-11-18T18:24:20,436 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731954260238.a244d0ab318eb4dd77bc5235828d9d51. 2024-11-18T18:24:20,436 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731954260238.a244d0ab318eb4dd77bc5235828d9d51. 2024-11-18T18:24:20,437 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731954260238.a244d0ab318eb4dd77bc5235828d9d51. after waiting 0 ms 2024-11-18T18:24:20,437 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731954260238.a244d0ab318eb4dd77bc5235828d9d51. 2024-11-18T18:24:20,437 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731954260238.a244d0ab318eb4dd77bc5235828d9d51. 2024-11-18T18:24:20,437 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for a244d0ab318eb4dd77bc5235828d9d51: Waiting for close lock at 1731954260436Disabling compacts and flushes for region at 1731954260436Disabling writes for close at 1731954260437 (+1 ms)Writing region close event to WAL at 1731954260437Closed at 1731954260437 2024-11-18T18:24:20,437 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => fb8fa4293ea5e58c18dbc5b978355ca9, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731954260238.fb8fa4293ea5e58c18dbc5b978355ca9.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93 2024-11-18T18:24:20,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741852_1028 (size=110) 2024-11-18T18:24:20,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741852_1028 (size=110) 2024-11-18T18:24:20,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741852_1028 (size=110) 2024-11-18T18:24:20,461 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731954260238.140f5077d3b730b643b558ebe8a13abd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:20,461 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 140f5077d3b730b643b558ebe8a13abd, disabling compactions & flushes 2024-11-18T18:24:20,461 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731954260238.140f5077d3b730b643b558ebe8a13abd. 2024-11-18T18:24:20,461 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731954260238.140f5077d3b730b643b558ebe8a13abd. 2024-11-18T18:24:20,461 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731954260238.140f5077d3b730b643b558ebe8a13abd. after waiting 0 ms 2024-11-18T18:24:20,461 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731954260238.140f5077d3b730b643b558ebe8a13abd. 2024-11-18T18:24:20,461 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731954260238.140f5077d3b730b643b558ebe8a13abd. 2024-11-18T18:24:20,462 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 140f5077d3b730b643b558ebe8a13abd: Waiting for close lock at 1731954260461Disabling compacts and flushes for region at 1731954260461Disabling writes for close at 1731954260461Writing region close event to WAL at 1731954260461Closed at 1731954260461 2024-11-18T18:24:20,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741853_1029 (size=110) 2024-11-18T18:24:20,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741854_1030 (size=110) 2024-11-18T18:24:20,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741854_1030 (size=110) 2024-11-18T18:24:20,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741853_1029 (size=110) 2024-11-18T18:24:20,463 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 69ae79c6f9eeed559fa88701f773706c, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731954260238.69ae79c6f9eeed559fa88701f773706c.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93 2024-11-18T18:24:20,465 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731954260238.fb8fa4293ea5e58c18dbc5b978355ca9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:20,465 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing fb8fa4293ea5e58c18dbc5b978355ca9, disabling compactions & flushes 2024-11-18T18:24:20,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741854_1030 (size=110) 2024-11-18T18:24:20,466 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731954260238.fb8fa4293ea5e58c18dbc5b978355ca9. 2024-11-18T18:24:20,466 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731954260238.fb8fa4293ea5e58c18dbc5b978355ca9. 2024-11-18T18:24:20,466 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731954260238.fb8fa4293ea5e58c18dbc5b978355ca9. after waiting 0 ms 2024-11-18T18:24:20,466 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731954260238.fb8fa4293ea5e58c18dbc5b978355ca9. 2024-11-18T18:24:20,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741853_1029 (size=110) 2024-11-18T18:24:20,466 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731954260238.fb8fa4293ea5e58c18dbc5b978355ca9. 2024-11-18T18:24:20,466 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for fb8fa4293ea5e58c18dbc5b978355ca9: Waiting for close lock at 1731954260465Disabling compacts and flushes for region at 1731954260465Disabling writes for close at 1731954260466 (+1 ms)Writing region close event to WAL at 1731954260466Closed at 1731954260466 2024-11-18T18:24:20,466 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 75a13ed51395b1016b90c6b0d90a7d0a, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731954260238.75a13ed51395b1016b90c6b0d90a7d0a.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93 2024-11-18T18:24:20,467 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731954260238.5e69ef2147d87d1b83dbcbb4705cbabe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:20,467 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing 5e69ef2147d87d1b83dbcbb4705cbabe, disabling compactions & flushes 2024-11-18T18:24:20,467 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731954260238.5e69ef2147d87d1b83dbcbb4705cbabe. 2024-11-18T18:24:20,467 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731954260238.5e69ef2147d87d1b83dbcbb4705cbabe. 2024-11-18T18:24:20,467 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731954260238.5e69ef2147d87d1b83dbcbb4705cbabe. after waiting 0 ms 2024-11-18T18:24:20,467 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731954260238.5e69ef2147d87d1b83dbcbb4705cbabe. 2024-11-18T18:24:20,467 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731954260238.5e69ef2147d87d1b83dbcbb4705cbabe. 2024-11-18T18:24:20,467 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for 5e69ef2147d87d1b83dbcbb4705cbabe: Waiting for close lock at 1731954260467Disabling compacts and flushes for region at 1731954260467Disabling writes for close at 1731954260467Writing region close event to WAL at 1731954260467Closed at 1731954260467 2024-11-18T18:24:20,468 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => aaf2435418703b2a31c1190af1560ba8, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731954260238.aaf2435418703b2a31c1190af1560ba8.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93 2024-11-18T18:24:20,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741855_1031 (size=110) 2024-11-18T18:24:20,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741855_1031 (size=110) 2024-11-18T18:24:20,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741855_1031 (size=110) 2024-11-18T18:24:20,489 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731954260238.69ae79c6f9eeed559fa88701f773706c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:20,489 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 69ae79c6f9eeed559fa88701f773706c, disabling compactions & flushes 2024-11-18T18:24:20,489 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731954260238.69ae79c6f9eeed559fa88701f773706c. 2024-11-18T18:24:20,489 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731954260238.69ae79c6f9eeed559fa88701f773706c. 2024-11-18T18:24:20,489 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731954260238.69ae79c6f9eeed559fa88701f773706c. after waiting 0 ms 2024-11-18T18:24:20,489 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731954260238.69ae79c6f9eeed559fa88701f773706c. 2024-11-18T18:24:20,489 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731954260238.69ae79c6f9eeed559fa88701f773706c. 2024-11-18T18:24:20,490 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 69ae79c6f9eeed559fa88701f773706c: Waiting for close lock at 1731954260489Disabling compacts and flushes for region at 1731954260489Disabling writes for close at 1731954260489Writing region close event to WAL at 1731954260489Closed at 1731954260489 2024-11-18T18:24:20,490 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d1ad74f79ba4d7fd1619c83a55106331, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731954260238.d1ad74f79ba4d7fd1619c83a55106331.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93 2024-11-18T18:24:20,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741856_1032 (size=110) 2024-11-18T18:24:20,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741856_1032 (size=110) 2024-11-18T18:24:20,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741856_1032 (size=110) 2024-11-18T18:24:20,493 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731954260238.75a13ed51395b1016b90c6b0d90a7d0a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:20,493 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing 75a13ed51395b1016b90c6b0d90a7d0a, disabling compactions & flushes 2024-11-18T18:24:20,493 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731954260238.75a13ed51395b1016b90c6b0d90a7d0a. 2024-11-18T18:24:20,493 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731954260238.75a13ed51395b1016b90c6b0d90a7d0a. 2024-11-18T18:24:20,493 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731954260238.75a13ed51395b1016b90c6b0d90a7d0a. after waiting 0 ms 2024-11-18T18:24:20,494 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731954260238.75a13ed51395b1016b90c6b0d90a7d0a. 2024-11-18T18:24:20,494 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731954260238.75a13ed51395b1016b90c6b0d90a7d0a. 2024-11-18T18:24:20,494 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for 75a13ed51395b1016b90c6b0d90a7d0a: Waiting for close lock at 1731954260493Disabling compacts and flushes for region at 1731954260493Disabling writes for close at 1731954260494 (+1 ms)Writing region close event to WAL at 1731954260494Closed at 1731954260494 2024-11-18T18:24:20,495 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 81f0db1bea1e11879dc1cafb06c0756d, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731954260238.81f0db1bea1e11879dc1cafb06c0756d.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93 2024-11-18T18:24:20,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741857_1033 (size=110) 2024-11-18T18:24:20,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741857_1033 (size=110) 2024-11-18T18:24:20,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741857_1033 (size=110) 2024-11-18T18:24:20,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741858_1034 (size=110) 2024-11-18T18:24:20,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741859_1035 (size=110) 2024-11-18T18:24:20,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741858_1034 (size=110) 2024-11-18T18:24:20,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741859_1035 (size=110) 2024-11-18T18:24:20,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741858_1034 (size=110) 2024-11-18T18:24:20,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741859_1035 (size=110) 2024-11-18T18:24:20,516 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731954260238.d1ad74f79ba4d7fd1619c83a55106331.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:20,516 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing d1ad74f79ba4d7fd1619c83a55106331, disabling compactions & flushes 2024-11-18T18:24:20,516 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731954260238.d1ad74f79ba4d7fd1619c83a55106331. 2024-11-18T18:24:20,516 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731954260238.d1ad74f79ba4d7fd1619c83a55106331. 2024-11-18T18:24:20,516 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731954260238.d1ad74f79ba4d7fd1619c83a55106331. after waiting 0 ms 2024-11-18T18:24:20,516 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731954260238.d1ad74f79ba4d7fd1619c83a55106331. 2024-11-18T18:24:20,516 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731954260238.d1ad74f79ba4d7fd1619c83a55106331. 2024-11-18T18:24:20,516 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for d1ad74f79ba4d7fd1619c83a55106331: Waiting for close lock at 1731954260516Disabling compacts and flushes for region at 1731954260516Disabling writes for close at 1731954260516Writing region close event to WAL at 1731954260516Closed at 1731954260516 2024-11-18T18:24:20,517 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2a8fcaf8905dcde0537806fb16b9ff4c, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731954260238.2a8fcaf8905dcde0537806fb16b9ff4c.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93 2024-11-18T18:24:20,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741860_1036 (size=110) 2024-11-18T18:24:20,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741860_1036 (size=110) 2024-11-18T18:24:20,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741860_1036 (size=110) 2024-11-18T18:24:20,528 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731954260238.2a8fcaf8905dcde0537806fb16b9ff4c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:20,529 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 2a8fcaf8905dcde0537806fb16b9ff4c, disabling compactions & flushes 2024-11-18T18:24:20,529 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731954260238.2a8fcaf8905dcde0537806fb16b9ff4c. 2024-11-18T18:24:20,529 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731954260238.2a8fcaf8905dcde0537806fb16b9ff4c. 2024-11-18T18:24:20,529 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731954260238.2a8fcaf8905dcde0537806fb16b9ff4c. after waiting 0 ms 2024-11-18T18:24:20,529 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731954260238.2a8fcaf8905dcde0537806fb16b9ff4c. 2024-11-18T18:24:20,529 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731954260238.2a8fcaf8905dcde0537806fb16b9ff4c. 2024-11-18T18:24:20,529 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2a8fcaf8905dcde0537806fb16b9ff4c: Waiting for close lock at 1731954260529Disabling compacts and flushes for region at 1731954260529Disabling writes for close at 1731954260529Writing region close event to WAL at 1731954260529Closed at 1731954260529 2024-11-18T18:24:20,529 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 579ee42a94597fdc1db9a4820a53ee64, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731954260238.579ee42a94597fdc1db9a4820a53ee64.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93 2024-11-18T18:24:20,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741861_1037 (size=110) 2024-11-18T18:24:20,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741861_1037 (size=110) 2024-11-18T18:24:20,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741861_1037 (size=110) 2024-11-18T18:24:20,540 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731954260238.579ee42a94597fdc1db9a4820a53ee64.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:20,540 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 579ee42a94597fdc1db9a4820a53ee64, disabling compactions & flushes 2024-11-18T18:24:20,540 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731954260238.579ee42a94597fdc1db9a4820a53ee64. 2024-11-18T18:24:20,540 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731954260238.579ee42a94597fdc1db9a4820a53ee64. 2024-11-18T18:24:20,540 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731954260238.579ee42a94597fdc1db9a4820a53ee64. after waiting 0 ms 2024-11-18T18:24:20,540 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731954260238.579ee42a94597fdc1db9a4820a53ee64. 2024-11-18T18:24:20,540 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731954260238.579ee42a94597fdc1db9a4820a53ee64. 2024-11-18T18:24:20,540 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 579ee42a94597fdc1db9a4820a53ee64: Waiting for close lock at 1731954260540Disabling compacts and flushes for region at 1731954260540Disabling writes for close at 1731954260540Writing region close event to WAL at 1731954260540Closed at 1731954260540 2024-11-18T18:24:20,541 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 779798cd60db955ed78a4329cac08969, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731954260238.779798cd60db955ed78a4329cac08969.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93 2024-11-18T18:24:20,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741862_1038 (size=110) 2024-11-18T18:24:20,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741862_1038 (size=110) 2024-11-18T18:24:20,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741862_1038 (size=110) 2024-11-18T18:24:20,553 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731954260238.779798cd60db955ed78a4329cac08969.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:20,553 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 779798cd60db955ed78a4329cac08969, disabling compactions & flushes 2024-11-18T18:24:20,553 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731954260238.779798cd60db955ed78a4329cac08969. 2024-11-18T18:24:20,553 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731954260238.779798cd60db955ed78a4329cac08969. 2024-11-18T18:24:20,553 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731954260238.779798cd60db955ed78a4329cac08969. after waiting 0 ms 2024-11-18T18:24:20,553 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731954260238.779798cd60db955ed78a4329cac08969. 2024-11-18T18:24:20,553 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731954260238.779798cd60db955ed78a4329cac08969. 2024-11-18T18:24:20,553 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 779798cd60db955ed78a4329cac08969: Waiting for close lock at 1731954260553Disabling compacts and flushes for region at 1731954260553Disabling writes for close at 1731954260553Writing region close event to WAL at 1731954260553Closed at 1731954260553 2024-11-18T18:24:20,554 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 08ccc32db22fb75f5466ad774ad9e8ba, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731954260238.08ccc32db22fb75f5466ad774ad9e8ba.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93 2024-11-18T18:24:20,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741863_1039 (size=107) 2024-11-18T18:24:20,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741863_1039 (size=107) 2024-11-18T18:24:20,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741863_1039 (size=107) 2024-11-18T18:24:20,566 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731954260238.08ccc32db22fb75f5466ad774ad9e8ba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:20,566 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 08ccc32db22fb75f5466ad774ad9e8ba, disabling compactions & flushes 2024-11-18T18:24:20,566 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731954260238.08ccc32db22fb75f5466ad774ad9e8ba. 2024-11-18T18:24:20,566 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731954260238.08ccc32db22fb75f5466ad774ad9e8ba. 2024-11-18T18:24:20,567 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731954260238.08ccc32db22fb75f5466ad774ad9e8ba. after waiting 0 ms 2024-11-18T18:24:20,567 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731954260238.08ccc32db22fb75f5466ad774ad9e8ba. 2024-11-18T18:24:20,567 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731954260238.08ccc32db22fb75f5466ad774ad9e8ba. 2024-11-18T18:24:20,567 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 08ccc32db22fb75f5466ad774ad9e8ba: Waiting for close lock at 1731954260566Disabling compacts and flushes for region at 1731954260566Disabling writes for close at 1731954260567 (+1 ms)Writing region close event to WAL at 1731954260567Closed at 1731954260567 2024-11-18T18:24:20,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T18:24:20,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T18:24:20,904 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731954260238.aaf2435418703b2a31c1190af1560ba8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:20,904 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing aaf2435418703b2a31c1190af1560ba8, disabling compactions & flushes 2024-11-18T18:24:20,904 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731954260238.aaf2435418703b2a31c1190af1560ba8. 2024-11-18T18:24:20,904 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731954260238.aaf2435418703b2a31c1190af1560ba8. 2024-11-18T18:24:20,904 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731954260238.aaf2435418703b2a31c1190af1560ba8. after waiting 0 ms 2024-11-18T18:24:20,904 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731954260238.aaf2435418703b2a31c1190af1560ba8. 2024-11-18T18:24:20,904 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731954260238.aaf2435418703b2a31c1190af1560ba8. 2024-11-18T18:24:20,905 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for aaf2435418703b2a31c1190af1560ba8: Waiting for close lock at 1731954260904Disabling compacts and flushes for region at 1731954260904Disabling writes for close at 1731954260904Writing region close event to WAL at 1731954260904Closed at 1731954260904 2024-11-18T18:24:20,915 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731954260238.81f0db1bea1e11879dc1cafb06c0756d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:20,915 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing 81f0db1bea1e11879dc1cafb06c0756d, disabling compactions & flushes 2024-11-18T18:24:20,915 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731954260238.81f0db1bea1e11879dc1cafb06c0756d. 2024-11-18T18:24:20,915 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731954260238.81f0db1bea1e11879dc1cafb06c0756d. 2024-11-18T18:24:20,915 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731954260238.81f0db1bea1e11879dc1cafb06c0756d. after waiting 0 ms 2024-11-18T18:24:20,915 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731954260238.81f0db1bea1e11879dc1cafb06c0756d. 2024-11-18T18:24:20,915 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731954260238.81f0db1bea1e11879dc1cafb06c0756d. 2024-11-18T18:24:20,916 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for 81f0db1bea1e11879dc1cafb06c0756d: Waiting for close lock at 1731954260915Disabling compacts and flushes for region at 1731954260915Disabling writes for close at 1731954260915Writing region close event to WAL at 1731954260915Closed at 1731954260915 2024-11-18T18:24:20,919 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T18:24:20,925 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731954260238.22971deb0db38b7cccde5c3bc0868f22.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731954260919"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731954260919"}]},"ts":"1731954260919"} 2024-11-18T18:24:20,925 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731954260238.43138a6a12c2efea7093064622bb3cd8.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731954260919"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731954260919"}]},"ts":"1731954260919"} 2024-11-18T18:24:20,926 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731954260238.8bb7ae0747eda2290799ca8fa83f9ede.","families":{"info":[{"qualifier":"regioninfo","vlen":106,"tag":[],"timestamp":"1731954260919"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731954260919"}]},"ts":"1731954260919"} 2024-11-18T18:24:20,926 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731954260238.bb099f0975ebdb618ef8e3b2c263bbca.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731954260919"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731954260919"}]},"ts":"1731954260919"} 2024-11-18T18:24:20,926 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731954260238.bcf8695060a326b7d24615ba6cb932b1.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731954260919"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731954260919"}]},"ts":"1731954260919"} 2024-11-18T18:24:20,926 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731954260238.fb91ff28ef3c18f0dc0d3f26aac68eca.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731954260919"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731954260919"}]},"ts":"1731954260919"} 2024-11-18T18:24:20,927 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731954260238.c8b4dd2a6d8dd59085c547b55aac0de6.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731954260919"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731954260919"}]},"ts":"1731954260919"} 2024-11-18T18:24:20,927 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731954260238.40a213dc6b2a0fd809c71612c437ab83.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731954260919"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731954260919"}]},"ts":"1731954260919"} 2024-11-18T18:24:20,927 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731954260238.f39571a93e25091090ea0831d54936b5.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731954260919"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731954260919"}]},"ts":"1731954260919"} 2024-11-18T18:24:20,927 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731954260238.97a938e093c14701871e85a1fbb6e07d.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731954260919"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731954260919"}]},"ts":"1731954260919"} 2024-11-18T18:24:20,927 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731954260238.c3e2f3fb6516fcaaef4d0a2c8b63842d.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731954260919"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731954260919"}]},"ts":"1731954260919"} 2024-11-18T18:24:20,927 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731954260238.97377a91d35f8b242120f603c456c41e.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731954260919"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731954260919"}]},"ts":"1731954260919"} 2024-11-18T18:24:20,927 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731954260238.012c277c99c564dd90e8e5f6850b6ce0.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731954260919"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731954260919"}]},"ts":"1731954260919"} 2024-11-18T18:24:20,928 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731954260238.a244d0ab318eb4dd77bc5235828d9d51.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731954260919"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731954260919"}]},"ts":"1731954260919"} 2024-11-18T18:24:20,928 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731954260238.140f5077d3b730b643b558ebe8a13abd.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731954260919"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731954260919"}]},"ts":"1731954260919"} 2024-11-18T18:24:20,928 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731954260238.fb8fa4293ea5e58c18dbc5b978355ca9.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731954260919"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731954260919"}]},"ts":"1731954260919"} 2024-11-18T18:24:20,928 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731954260238.5e69ef2147d87d1b83dbcbb4705cbabe.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731954260919"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731954260919"}]},"ts":"1731954260919"} 2024-11-18T18:24:20,928 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731954260238.69ae79c6f9eeed559fa88701f773706c.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731954260919"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731954260919"}]},"ts":"1731954260919"} 2024-11-18T18:24:20,928 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731954260238.75a13ed51395b1016b90c6b0d90a7d0a.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731954260919"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731954260919"}]},"ts":"1731954260919"} 2024-11-18T18:24:20,929 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731954260238.d1ad74f79ba4d7fd1619c83a55106331.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731954260919"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731954260919"}]},"ts":"1731954260919"} 2024-11-18T18:24:20,929 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731954260238.2a8fcaf8905dcde0537806fb16b9ff4c.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731954260919"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731954260919"}]},"ts":"1731954260919"} 2024-11-18T18:24:20,929 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731954260238.579ee42a94597fdc1db9a4820a53ee64.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731954260919"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731954260919"}]},"ts":"1731954260919"} 2024-11-18T18:24:20,929 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731954260238.779798cd60db955ed78a4329cac08969.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731954260919"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731954260919"}]},"ts":"1731954260919"} 2024-11-18T18:24:20,929 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731954260238.08ccc32db22fb75f5466ad774ad9e8ba.","families":{"info":[{"qualifier":"regioninfo","vlen":106,"tag":[],"timestamp":"1731954260919"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731954260919"}]},"ts":"1731954260919"} 2024-11-18T18:24:20,930 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731954260238.aaf2435418703b2a31c1190af1560ba8.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731954260919"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731954260919"}]},"ts":"1731954260919"} 2024-11-18T18:24:20,930 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731954260238.81f0db1bea1e11879dc1cafb06c0756d.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731954260919"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731954260919"}]},"ts":"1731954260919"} 2024-11-18T18:24:20,991 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-11-18T18:24:20,994 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T18:24:20,998 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731954260994"}]},"ts":"1731954260994"} 2024-11-18T18:24:21,003 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, state=ENABLING in hbase:meta 2024-11-18T18:24:21,005 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {961a81da9827=0} racks are {/default-rack=0} 2024-11-18T18:24:21,062 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-18T18:24:21,062 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-18T18:24:21,062 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-18T18:24:21,062 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-18T18:24:21,062 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-18T18:24:21,062 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-18T18:24:21,062 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-18T18:24:21,062 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-18T18:24:21,062 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-18T18:24:21,063 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-18T18:24:21,066 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8bb7ae0747eda2290799ca8fa83f9ede, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=22971deb0db38b7cccde5c3bc0868f22, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=43138a6a12c2efea7093064622bb3cd8, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=bb099f0975ebdb618ef8e3b2c263bbca, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=bcf8695060a326b7d24615ba6cb932b1, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fb91ff28ef3c18f0dc0d3f26aac68eca, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c8b4dd2a6d8dd59085c547b55aac0de6, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=40a213dc6b2a0fd809c71612c437ab83, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=f39571a93e25091090ea0831d54936b5, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=97a938e093c14701871e85a1fbb6e07d, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c3e2f3fb6516fcaaef4d0a2c8b63842d, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=97377a91d35f8b242120f603c456c41e, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a244d0ab318eb4dd77bc5235828d9d51, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=012c277c99c564dd90e8e5f6850b6ce0, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=140f5077d3b730b643b558ebe8a13abd, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5e69ef2147d87d1b83dbcbb4705cbabe, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fb8fa4293ea5e58c18dbc5b978355ca9, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=69ae79c6f9eeed559fa88701f773706c, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=75a13ed51395b1016b90c6b0d90a7d0a, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=aaf2435418703b2a31c1190af1560ba8, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d1ad74f79ba4d7fd1619c83a55106331, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=81f0db1bea1e11879dc1cafb06c0756d, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2a8fcaf8905dcde0537806fb16b9ff4c, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=579ee42a94597fdc1db9a4820a53ee64, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=779798cd60db955ed78a4329cac08969, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=08ccc32db22fb75f5466ad774ad9e8ba, ASSIGN}] 2024-11-18T18:24:21,079 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=08ccc32db22fb75f5466ad774ad9e8ba, ASSIGN 2024-11-18T18:24:21,080 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=08ccc32db22fb75f5466ad774ad9e8ba, ASSIGN; state=OFFLINE, location=961a81da9827,38709,1731954257723; forceNewPlan=false, retain=false 2024-11-18T18:24:21,082 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=779798cd60db955ed78a4329cac08969, ASSIGN 2024-11-18T18:24:21,084 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=779798cd60db955ed78a4329cac08969, ASSIGN; state=OFFLINE, location=961a81da9827,36379,1731954257826; forceNewPlan=false, retain=false 2024-11-18T18:24:21,086 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=579ee42a94597fdc1db9a4820a53ee64, ASSIGN 2024-11-18T18:24:21,090 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=579ee42a94597fdc1db9a4820a53ee64, ASSIGN; state=OFFLINE, location=961a81da9827,42895,1731954257867; forceNewPlan=false, retain=false 2024-11-18T18:24:21,092 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2a8fcaf8905dcde0537806fb16b9ff4c, ASSIGN 2024-11-18T18:24:21,093 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2a8fcaf8905dcde0537806fb16b9ff4c, ASSIGN; state=OFFLINE, location=961a81da9827,38709,1731954257723; forceNewPlan=false, retain=false 2024-11-18T18:24:21,095 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=81f0db1bea1e11879dc1cafb06c0756d, ASSIGN 2024-11-18T18:24:21,096 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=81f0db1bea1e11879dc1cafb06c0756d, ASSIGN; state=OFFLINE, location=961a81da9827,36379,1731954257826; forceNewPlan=false, retain=false 2024-11-18T18:24:21,098 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d1ad74f79ba4d7fd1619c83a55106331, ASSIGN 2024-11-18T18:24:21,100 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d1ad74f79ba4d7fd1619c83a55106331, ASSIGN; state=OFFLINE, location=961a81da9827,36379,1731954257826; forceNewPlan=false, retain=false 2024-11-18T18:24:21,101 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=aaf2435418703b2a31c1190af1560ba8, ASSIGN 2024-11-18T18:24:21,102 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=aaf2435418703b2a31c1190af1560ba8, ASSIGN; state=OFFLINE, location=961a81da9827,42895,1731954257867; forceNewPlan=false, retain=false 2024-11-18T18:24:21,104 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=75a13ed51395b1016b90c6b0d90a7d0a, ASSIGN 2024-11-18T18:24:21,105 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=75a13ed51395b1016b90c6b0d90a7d0a, ASSIGN; state=OFFLINE, location=961a81da9827,42895,1731954257867; forceNewPlan=false, retain=false 2024-11-18T18:24:21,106 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=69ae79c6f9eeed559fa88701f773706c, ASSIGN 2024-11-18T18:24:21,108 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=69ae79c6f9eeed559fa88701f773706c, ASSIGN; state=OFFLINE, location=961a81da9827,38709,1731954257723; forceNewPlan=false, retain=false 2024-11-18T18:24:21,109 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fb8fa4293ea5e58c18dbc5b978355ca9, ASSIGN 2024-11-18T18:24:21,111 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fb8fa4293ea5e58c18dbc5b978355ca9, ASSIGN; state=OFFLINE, location=961a81da9827,42895,1731954257867; forceNewPlan=false, retain=false 2024-11-18T18:24:21,112 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5e69ef2147d87d1b83dbcbb4705cbabe, ASSIGN 2024-11-18T18:24:21,113 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5e69ef2147d87d1b83dbcbb4705cbabe, ASSIGN; state=OFFLINE, location=961a81da9827,36379,1731954257826; forceNewPlan=false, retain=false 2024-11-18T18:24:21,114 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=140f5077d3b730b643b558ebe8a13abd, ASSIGN 2024-11-18T18:24:21,115 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=140f5077d3b730b643b558ebe8a13abd, ASSIGN; state=OFFLINE, location=961a81da9827,38709,1731954257723; forceNewPlan=false, retain=false 2024-11-18T18:24:21,117 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=012c277c99c564dd90e8e5f6850b6ce0, ASSIGN 2024-11-18T18:24:21,118 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=012c277c99c564dd90e8e5f6850b6ce0, ASSIGN; state=OFFLINE, location=961a81da9827,42895,1731954257867; forceNewPlan=false, retain=false 2024-11-18T18:24:21,120 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a244d0ab318eb4dd77bc5235828d9d51, ASSIGN 2024-11-18T18:24:21,121 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a244d0ab318eb4dd77bc5235828d9d51, ASSIGN; state=OFFLINE, location=961a81da9827,36379,1731954257826; forceNewPlan=false, retain=false 2024-11-18T18:24:21,123 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=97377a91d35f8b242120f603c456c41e, ASSIGN 2024-11-18T18:24:21,124 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=97377a91d35f8b242120f603c456c41e, ASSIGN; state=OFFLINE, location=961a81da9827,38709,1731954257723; forceNewPlan=false, retain=false 2024-11-18T18:24:21,126 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c3e2f3fb6516fcaaef4d0a2c8b63842d, ASSIGN 2024-11-18T18:24:21,127 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c3e2f3fb6516fcaaef4d0a2c8b63842d, ASSIGN; state=OFFLINE, location=961a81da9827,36379,1731954257826; forceNewPlan=false, retain=false 2024-11-18T18:24:21,128 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=97a938e093c14701871e85a1fbb6e07d, ASSIGN 2024-11-18T18:24:21,131 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=97a938e093c14701871e85a1fbb6e07d, ASSIGN; state=OFFLINE, location=961a81da9827,42895,1731954257867; forceNewPlan=false, retain=false 2024-11-18T18:24:21,133 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=f39571a93e25091090ea0831d54936b5, ASSIGN 2024-11-18T18:24:21,134 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=f39571a93e25091090ea0831d54936b5, ASSIGN; state=OFFLINE, location=961a81da9827,38709,1731954257723; forceNewPlan=false, retain=false 2024-11-18T18:24:21,135 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=40a213dc6b2a0fd809c71612c437ab83, ASSIGN 2024-11-18T18:24:21,137 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=40a213dc6b2a0fd809c71612c437ab83, ASSIGN; state=OFFLINE, location=961a81da9827,36379,1731954257826; forceNewPlan=false, retain=false 2024-11-18T18:24:21,138 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c8b4dd2a6d8dd59085c547b55aac0de6, ASSIGN 2024-11-18T18:24:21,140 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c8b4dd2a6d8dd59085c547b55aac0de6, ASSIGN; state=OFFLINE, location=961a81da9827,42895,1731954257867; forceNewPlan=false, retain=false 2024-11-18T18:24:21,141 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fb91ff28ef3c18f0dc0d3f26aac68eca, ASSIGN 2024-11-18T18:24:21,143 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fb91ff28ef3c18f0dc0d3f26aac68eca, ASSIGN; state=OFFLINE, location=961a81da9827,38709,1731954257723; forceNewPlan=false, retain=false 2024-11-18T18:24:21,144 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=bcf8695060a326b7d24615ba6cb932b1, ASSIGN 2024-11-18T18:24:21,146 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=bcf8695060a326b7d24615ba6cb932b1, ASSIGN; state=OFFLINE, location=961a81da9827,36379,1731954257826; forceNewPlan=false, retain=false 2024-11-18T18:24:21,147 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=bb099f0975ebdb618ef8e3b2c263bbca, ASSIGN 2024-11-18T18:24:21,149 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=bb099f0975ebdb618ef8e3b2c263bbca, ASSIGN; state=OFFLINE, location=961a81da9827,42895,1731954257867; forceNewPlan=false, retain=false 2024-11-18T18:24:21,150 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=43138a6a12c2efea7093064622bb3cd8, ASSIGN 2024-11-18T18:24:21,151 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=43138a6a12c2efea7093064622bb3cd8, ASSIGN; state=OFFLINE, location=961a81da9827,36379,1731954257826; forceNewPlan=false, retain=false 2024-11-18T18:24:21,153 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=22971deb0db38b7cccde5c3bc0868f22, ASSIGN 2024-11-18T18:24:21,154 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=22971deb0db38b7cccde5c3bc0868f22, ASSIGN; state=OFFLINE, location=961a81da9827,42895,1731954257867; forceNewPlan=false, retain=false 2024-11-18T18:24:21,156 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8bb7ae0747eda2290799ca8fa83f9ede, ASSIGN 2024-11-18T18:24:21,157 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8bb7ae0747eda2290799ca8fa83f9ede, ASSIGN; state=OFFLINE, location=961a81da9827,38709,1731954257723; forceNewPlan=false, retain=false 2024-11-18T18:24:21,234 INFO [961a81da9827:33089 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-11-18T18:24:21,235 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=579ee42a94597fdc1db9a4820a53ee64, regionState=OPENING, regionLocation=961a81da9827,42895,1731954257867 2024-11-18T18:24:21,236 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=aaf2435418703b2a31c1190af1560ba8, regionState=OPENING, regionLocation=961a81da9827,42895,1731954257867 2024-11-18T18:24:21,237 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=75a13ed51395b1016b90c6b0d90a7d0a, regionState=OPENING, regionLocation=961a81da9827,42895,1731954257867 2024-11-18T18:24:21,238 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=fb8fa4293ea5e58c18dbc5b978355ca9, regionState=OPENING, regionLocation=961a81da9827,42895,1731954257867 2024-11-18T18:24:21,239 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=012c277c99c564dd90e8e5f6850b6ce0, regionState=OPENING, regionLocation=961a81da9827,42895,1731954257867 2024-11-18T18:24:21,239 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=97a938e093c14701871e85a1fbb6e07d, regionState=OPENING, regionLocation=961a81da9827,42895,1731954257867 2024-11-18T18:24:21,240 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=c8b4dd2a6d8dd59085c547b55aac0de6, regionState=OPENING, regionLocation=961a81da9827,42895,1731954257867 2024-11-18T18:24:21,240 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=579ee42a94597fdc1db9a4820a53ee64, ASSIGN because future has completed 2024-11-18T18:24:21,240 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=bb099f0975ebdb618ef8e3b2c263bbca, regionState=OPENING, regionLocation=961a81da9827,42895,1731954257867 2024-11-18T18:24:21,241 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=22971deb0db38b7cccde5c3bc0868f22, regionState=OPENING, regionLocation=961a81da9827,42895,1731954257867 2024-11-18T18:24:21,242 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 579ee42a94597fdc1db9a4820a53ee64, server=961a81da9827,42895,1731954257867}] 2024-11-18T18:24:21,244 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=aaf2435418703b2a31c1190af1560ba8, ASSIGN because future has completed 2024-11-18T18:24:21,246 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=75a13ed51395b1016b90c6b0d90a7d0a, ASSIGN because future has completed 2024-11-18T18:24:21,252 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fb8fa4293ea5e58c18dbc5b978355ca9, ASSIGN because future has completed 2024-11-18T18:24:21,253 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure fb8fa4293ea5e58c18dbc5b978355ca9, server=961a81da9827,42895,1731954257867}] 2024-11-18T18:24:21,254 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=012c277c99c564dd90e8e5f6850b6ce0, ASSIGN because future has completed 2024-11-18T18:24:21,256 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=97a938e093c14701871e85a1fbb6e07d, ASSIGN because future has completed 2024-11-18T18:24:21,258 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c8b4dd2a6d8dd59085c547b55aac0de6, ASSIGN because future has completed 2024-11-18T18:24:21,260 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=bb099f0975ebdb618ef8e3b2c263bbca, ASSIGN because future has completed 2024-11-18T18:24:21,261 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure c8b4dd2a6d8dd59085c547b55aac0de6, server=961a81da9827,42895,1731954257867}] 2024-11-18T18:24:21,263 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=22971deb0db38b7cccde5c3bc0868f22, ASSIGN because future has completed 2024-11-18T18:24:21,266 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 22971deb0db38b7cccde5c3bc0868f22, server=961a81da9827,42895,1731954257867}] 2024-11-18T18:24:21,271 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure bb099f0975ebdb618ef8e3b2c263bbca, server=961a81da9827,42895,1731954257867}] 2024-11-18T18:24:21,277 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 97a938e093c14701871e85a1fbb6e07d, server=961a81da9827,42895,1731954257867}] 2024-11-18T18:24:21,282 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 012c277c99c564dd90e8e5f6850b6ce0, server=961a81da9827,42895,1731954257867}] 2024-11-18T18:24:21,288 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 75a13ed51395b1016b90c6b0d90a7d0a, server=961a81da9827,42895,1731954257867}] 2024-11-18T18:24:21,294 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure aaf2435418703b2a31c1190af1560ba8, server=961a81da9827,42895,1731954257867}] 2024-11-18T18:24:21,300 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=08ccc32db22fb75f5466ad774ad9e8ba, regionState=OPENING, regionLocation=961a81da9827,38709,1731954257723 2024-11-18T18:24:21,301 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=2a8fcaf8905dcde0537806fb16b9ff4c, regionState=OPENING, regionLocation=961a81da9827,38709,1731954257723 2024-11-18T18:24:21,302 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=69ae79c6f9eeed559fa88701f773706c, regionState=OPENING, regionLocation=961a81da9827,38709,1731954257723 2024-11-18T18:24:21,303 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=140f5077d3b730b643b558ebe8a13abd, regionState=OPENING, regionLocation=961a81da9827,38709,1731954257723 2024-11-18T18:24:21,303 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=08ccc32db22fb75f5466ad774ad9e8ba, ASSIGN because future has completed 2024-11-18T18:24:21,304 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=97377a91d35f8b242120f603c456c41e, regionState=OPENING, regionLocation=961a81da9827,38709,1731954257723 2024-11-18T18:24:21,304 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 08ccc32db22fb75f5466ad774ad9e8ba, server=961a81da9827,38709,1731954257723}] 2024-11-18T18:24:21,305 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2a8fcaf8905dcde0537806fb16b9ff4c, ASSIGN because future has completed 2024-11-18T18:24:21,307 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=69ae79c6f9eeed559fa88701f773706c, ASSIGN because future has completed 2024-11-18T18:24:21,309 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=140f5077d3b730b643b558ebe8a13abd, ASSIGN because future has completed 2024-11-18T18:24:21,311 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=97377a91d35f8b242120f603c456c41e, ASSIGN because future has completed 2024-11-18T18:24:21,312 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 97377a91d35f8b242120f603c456c41e, server=961a81da9827,38709,1731954257723}] 2024-11-18T18:24:21,316 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 140f5077d3b730b643b558ebe8a13abd, server=961a81da9827,38709,1731954257723}] 2024-11-18T18:24:21,320 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 69ae79c6f9eeed559fa88701f773706c, server=961a81da9827,38709,1731954257723}] 2024-11-18T18:24:21,325 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2a8fcaf8905dcde0537806fb16b9ff4c, server=961a81da9827,38709,1731954257723}] 2024-11-18T18:24:21,332 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=f39571a93e25091090ea0831d54936b5, regionState=OPENING, regionLocation=961a81da9827,38709,1731954257723 2024-11-18T18:24:21,333 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=fb91ff28ef3c18f0dc0d3f26aac68eca, regionState=OPENING, regionLocation=961a81da9827,38709,1731954257723 2024-11-18T18:24:21,334 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8bb7ae0747eda2290799ca8fa83f9ede, regionState=OPENING, regionLocation=961a81da9827,38709,1731954257723 2024-11-18T18:24:21,334 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=779798cd60db955ed78a4329cac08969, regionState=OPENING, regionLocation=961a81da9827,36379,1731954257826 2024-11-18T18:24:21,335 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=81f0db1bea1e11879dc1cafb06c0756d, regionState=OPENING, regionLocation=961a81da9827,36379,1731954257826 2024-11-18T18:24:21,336 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=d1ad74f79ba4d7fd1619c83a55106331, regionState=OPENING, regionLocation=961a81da9827,36379,1731954257826 2024-11-18T18:24:21,337 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=5e69ef2147d87d1b83dbcbb4705cbabe, regionState=OPENING, regionLocation=961a81da9827,36379,1731954257826 2024-11-18T18:24:21,337 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=a244d0ab318eb4dd77bc5235828d9d51, regionState=OPENING, regionLocation=961a81da9827,36379,1731954257826 2024-11-18T18:24:21,338 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=c3e2f3fb6516fcaaef4d0a2c8b63842d, regionState=OPENING, regionLocation=961a81da9827,36379,1731954257826 2024-11-18T18:24:21,339 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=40a213dc6b2a0fd809c71612c437ab83, regionState=OPENING, regionLocation=961a81da9827,36379,1731954257826 2024-11-18T18:24:21,339 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=bcf8695060a326b7d24615ba6cb932b1, regionState=OPENING, regionLocation=961a81da9827,36379,1731954257826 2024-11-18T18:24:21,340 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=43138a6a12c2efea7093064622bb3cd8, regionState=OPENING, regionLocation=961a81da9827,36379,1731954257826 2024-11-18T18:24:21,341 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=f39571a93e25091090ea0831d54936b5, ASSIGN because future has completed 2024-11-18T18:24:21,341 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure f39571a93e25091090ea0831d54936b5, server=961a81da9827,38709,1731954257723}] 2024-11-18T18:24:21,344 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fb91ff28ef3c18f0dc0d3f26aac68eca, ASSIGN because future has completed 2024-11-18T18:24:21,346 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8bb7ae0747eda2290799ca8fa83f9ede, ASSIGN because future has completed 2024-11-18T18:24:21,347 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=779798cd60db955ed78a4329cac08969, ASSIGN because future has completed 2024-11-18T18:24:21,348 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 779798cd60db955ed78a4329cac08969, server=961a81da9827,36379,1731954257826}] 2024-11-18T18:24:21,349 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=81f0db1bea1e11879dc1cafb06c0756d, ASSIGN because future has completed 2024-11-18T18:24:21,351 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d1ad74f79ba4d7fd1619c83a55106331, ASSIGN because future has completed 2024-11-18T18:24:21,352 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5e69ef2147d87d1b83dbcbb4705cbabe, ASSIGN because future has completed 2024-11-18T18:24:21,354 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a244d0ab318eb4dd77bc5235828d9d51, ASSIGN because future has completed 2024-11-18T18:24:21,354 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure a244d0ab318eb4dd77bc5235828d9d51, server=961a81da9827,36379,1731954257826}] 2024-11-18T18:24:21,355 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c3e2f3fb6516fcaaef4d0a2c8b63842d, ASSIGN because future has completed 2024-11-18T18:24:21,358 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=40a213dc6b2a0fd809c71612c437ab83, ASSIGN because future has completed 2024-11-18T18:24:21,359 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=bcf8695060a326b7d24615ba6cb932b1, ASSIGN because future has completed 2024-11-18T18:24:21,361 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=43138a6a12c2efea7093064622bb3cd8, ASSIGN because future has completed 2024-11-18T18:24:21,361 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 43138a6a12c2efea7093064622bb3cd8, server=961a81da9827,36379,1731954257826}] 2024-11-18T18:24:21,365 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure bcf8695060a326b7d24615ba6cb932b1, server=961a81da9827,36379,1731954257826}] 2024-11-18T18:24:21,370 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 40a213dc6b2a0fd809c71612c437ab83, server=961a81da9827,36379,1731954257826}] 2024-11-18T18:24:21,374 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure c3e2f3fb6516fcaaef4d0a2c8b63842d, server=961a81da9827,36379,1731954257826}] 2024-11-18T18:24:21,379 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5e69ef2147d87d1b83dbcbb4705cbabe, server=961a81da9827,36379,1731954257826}] 2024-11-18T18:24:21,384 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1ad74f79ba4d7fd1619c83a55106331, server=961a81da9827,36379,1731954257826}] 2024-11-18T18:24:21,389 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 81f0db1bea1e11879dc1cafb06c0756d, server=961a81da9827,36379,1731954257826}] 2024-11-18T18:24:21,393 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8bb7ae0747eda2290799ca8fa83f9ede, server=961a81da9827,38709,1731954257723}] 2024-11-18T18:24:21,397 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-18T18:24:21,399 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure fb91ff28ef3c18f0dc0d3f26aac68eca, server=961a81da9827,38709,1731954257723}] 2024-11-18T18:24:21,400 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51827, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-18T18:24:21,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T18:24:21,409 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731954260238.97a938e093c14701871e85a1fbb6e07d. 2024-11-18T18:24:21,410 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 97a938e093c14701871e85a1fbb6e07d, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731954260238.97a938e093c14701871e85a1fbb6e07d.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-18T18:24:21,410 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 97a938e093c14701871e85a1fbb6e07d 2024-11-18T18:24:21,410 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731954260238.97a938e093c14701871e85a1fbb6e07d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:21,410 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 97a938e093c14701871e85a1fbb6e07d 2024-11-18T18:24:21,410 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 97a938e093c14701871e85a1fbb6e07d 2024-11-18T18:24:21,413 INFO [StoreOpener-97a938e093c14701871e85a1fbb6e07d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 97a938e093c14701871e85a1fbb6e07d 2024-11-18T18:24:21,415 INFO [StoreOpener-97a938e093c14701871e85a1fbb6e07d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97a938e093c14701871e85a1fbb6e07d columnFamilyName colfamily11 2024-11-18T18:24:21,415 DEBUG [StoreOpener-97a938e093c14701871e85a1fbb6e07d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,416 INFO [StoreOpener-97a938e093c14701871e85a1fbb6e07d-1 {}] regionserver.HStore(327): Store=97a938e093c14701871e85a1fbb6e07d/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,416 INFO [StoreOpener-97a938e093c14701871e85a1fbb6e07d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 97a938e093c14701871e85a1fbb6e07d 2024-11-18T18:24:21,418 INFO [StoreOpener-97a938e093c14701871e85a1fbb6e07d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97a938e093c14701871e85a1fbb6e07d columnFamilyName colfamily21 2024-11-18T18:24:21,418 DEBUG [StoreOpener-97a938e093c14701871e85a1fbb6e07d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,419 INFO [StoreOpener-97a938e093c14701871e85a1fbb6e07d-1 {}] regionserver.HStore(327): Store=97a938e093c14701871e85a1fbb6e07d/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,419 INFO [StoreOpener-97a938e093c14701871e85a1fbb6e07d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 97a938e093c14701871e85a1fbb6e07d 2024-11-18T18:24:21,421 INFO [StoreOpener-97a938e093c14701871e85a1fbb6e07d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97a938e093c14701871e85a1fbb6e07d columnFamilyName colfamily31 2024-11-18T18:24:21,421 DEBUG [StoreOpener-97a938e093c14701871e85a1fbb6e07d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,421 INFO [StoreOpener-97a938e093c14701871e85a1fbb6e07d-1 {}] regionserver.HStore(327): Store=97a938e093c14701871e85a1fbb6e07d/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,422 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 97a938e093c14701871e85a1fbb6e07d 2024-11-18T18:24:21,423 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/97a938e093c14701871e85a1fbb6e07d 2024-11-18T18:24:21,424 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/97a938e093c14701871e85a1fbb6e07d 2024-11-18T18:24:21,425 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 97a938e093c14701871e85a1fbb6e07d 2024-11-18T18:24:21,425 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 97a938e093c14701871e85a1fbb6e07d 2024-11-18T18:24:21,426 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T18:24:21,428 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 97a938e093c14701871e85a1fbb6e07d 2024-11-18T18:24:21,431 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/97a938e093c14701871e85a1fbb6e07d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T18:24:21,432 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 97a938e093c14701871e85a1fbb6e07d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68261699, jitterRate=0.017178580164909363}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T18:24:21,432 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 97a938e093c14701871e85a1fbb6e07d 2024-11-18T18:24:21,433 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 97a938e093c14701871e85a1fbb6e07d: Running coprocessor pre-open hook at 1731954261411Writing region info on filesystem at 1731954261411Initializing all the Stores at 1731954261412 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261412Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261412Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261413 (+1 ms)Cleaning up temporary data from old regions at 1731954261425 (+12 ms)Running coprocessor post-open hooks at 1731954261432 (+7 ms)Region opened successfully at 1731954261433 (+1 ms) 2024-11-18T18:24:21,435 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731954260238.97a938e093c14701871e85a1fbb6e07d., pid=36, masterSystemTime=1731954261396 2024-11-18T18:24:21,441 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731954260238.97a938e093c14701871e85a1fbb6e07d. 2024-11-18T18:24:21,441 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731954260238.97a938e093c14701871e85a1fbb6e07d. 2024-11-18T18:24:21,441 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731954260238.012c277c99c564dd90e8e5f6850b6ce0. 2024-11-18T18:24:21,441 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 012c277c99c564dd90e8e5f6850b6ce0, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731954260238.012c277c99c564dd90e8e5f6850b6ce0.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-18T18:24:21,442 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=97a938e093c14701871e85a1fbb6e07d, regionState=OPEN, openSeqNum=2, regionLocation=961a81da9827,42895,1731954257867 2024-11-18T18:24:21,442 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 012c277c99c564dd90e8e5f6850b6ce0 2024-11-18T18:24:21,442 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731954260238.012c277c99c564dd90e8e5f6850b6ce0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:21,442 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 012c277c99c564dd90e8e5f6850b6ce0 2024-11-18T18:24:21,442 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 012c277c99c564dd90e8e5f6850b6ce0 2024-11-18T18:24:21,445 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33089 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=961a81da9827,42895,1731954257867, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=97a938e093c14701871e85a1fbb6e07d. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-18T18:24:21,446 INFO [StoreOpener-012c277c99c564dd90e8e5f6850b6ce0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 012c277c99c564dd90e8e5f6850b6ce0 2024-11-18T18:24:21,447 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 97a938e093c14701871e85a1fbb6e07d, server=961a81da9827,42895,1731954257867 because future has completed 2024-11-18T18:24:21,450 INFO [StoreOpener-012c277c99c564dd90e8e5f6850b6ce0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 012c277c99c564dd90e8e5f6850b6ce0 columnFamilyName colfamily11 2024-11-18T18:24:21,450 DEBUG [StoreOpener-012c277c99c564dd90e8e5f6850b6ce0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,451 INFO [StoreOpener-012c277c99c564dd90e8e5f6850b6ce0-1 {}] regionserver.HStore(327): Store=012c277c99c564dd90e8e5f6850b6ce0/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,451 INFO [StoreOpener-012c277c99c564dd90e8e5f6850b6ce0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 012c277c99c564dd90e8e5f6850b6ce0 2024-11-18T18:24:21,452 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=14 2024-11-18T18:24:21,452 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 97a938e093c14701871e85a1fbb6e07d, server=961a81da9827,42895,1731954257867 in 172 msec 2024-11-18T18:24:21,453 INFO [StoreOpener-012c277c99c564dd90e8e5f6850b6ce0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 012c277c99c564dd90e8e5f6850b6ce0 columnFamilyName colfamily21 2024-11-18T18:24:21,453 DEBUG [StoreOpener-012c277c99c564dd90e8e5f6850b6ce0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,454 INFO [StoreOpener-012c277c99c564dd90e8e5f6850b6ce0-1 {}] regionserver.HStore(327): Store=012c277c99c564dd90e8e5f6850b6ce0/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,454 INFO [StoreOpener-012c277c99c564dd90e8e5f6850b6ce0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 012c277c99c564dd90e8e5f6850b6ce0 2024-11-18T18:24:21,455 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=97a938e093c14701871e85a1fbb6e07d, ASSIGN in 387 msec 2024-11-18T18:24:21,456 INFO [StoreOpener-012c277c99c564dd90e8e5f6850b6ce0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 012c277c99c564dd90e8e5f6850b6ce0 columnFamilyName colfamily31 2024-11-18T18:24:21,456 DEBUG [StoreOpener-012c277c99c564dd90e8e5f6850b6ce0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,457 INFO [StoreOpener-012c277c99c564dd90e8e5f6850b6ce0-1 {}] regionserver.HStore(327): Store=012c277c99c564dd90e8e5f6850b6ce0/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,457 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 012c277c99c564dd90e8e5f6850b6ce0 2024-11-18T18:24:21,458 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/012c277c99c564dd90e8e5f6850b6ce0 2024-11-18T18:24:21,458 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/012c277c99c564dd90e8e5f6850b6ce0 2024-11-18T18:24:21,459 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-18T18:24:21,460 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 012c277c99c564dd90e8e5f6850b6ce0 2024-11-18T18:24:21,461 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 012c277c99c564dd90e8e5f6850b6ce0 2024-11-18T18:24:21,461 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47917, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-18T18:24:21,462 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T18:24:21,464 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 012c277c99c564dd90e8e5f6850b6ce0 2024-11-18T18:24:21,466 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731954260238.2a8fcaf8905dcde0537806fb16b9ff4c. 2024-11-18T18:24:21,467 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => 2a8fcaf8905dcde0537806fb16b9ff4c, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731954260238.2a8fcaf8905dcde0537806fb16b9ff4c.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-18T18:24:21,467 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2a8fcaf8905dcde0537806fb16b9ff4c 2024-11-18T18:24:21,467 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731954260238.2a8fcaf8905dcde0537806fb16b9ff4c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:21,467 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for 2a8fcaf8905dcde0537806fb16b9ff4c 2024-11-18T18:24:21,467 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for 2a8fcaf8905dcde0537806fb16b9ff4c 2024-11-18T18:24:21,469 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/012c277c99c564dd90e8e5f6850b6ce0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T18:24:21,469 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 012c277c99c564dd90e8e5f6850b6ce0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72185566, jitterRate=0.07564875483512878}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T18:24:21,470 INFO [StoreOpener-2a8fcaf8905dcde0537806fb16b9ff4c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 2a8fcaf8905dcde0537806fb16b9ff4c 2024-11-18T18:24:21,470 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 012c277c99c564dd90e8e5f6850b6ce0 2024-11-18T18:24:21,470 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 012c277c99c564dd90e8e5f6850b6ce0: Running coprocessor pre-open hook at 1731954261442Writing region info on filesystem at 1731954261442Initializing all the Stores at 1731954261446 (+4 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261446Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261446Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261446Cleaning up temporary data from old regions at 1731954261461 (+15 ms)Running coprocessor post-open hooks at 1731954261470 (+9 ms)Region opened successfully at 1731954261470 2024-11-18T18:24:21,472 INFO [StoreOpener-2a8fcaf8905dcde0537806fb16b9ff4c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2a8fcaf8905dcde0537806fb16b9ff4c columnFamilyName colfamily11 2024-11-18T18:24:21,472 DEBUG [StoreOpener-2a8fcaf8905dcde0537806fb16b9ff4c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,472 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731954260238.012c277c99c564dd90e8e5f6850b6ce0., pid=37, masterSystemTime=1731954261396 2024-11-18T18:24:21,473 INFO [StoreOpener-2a8fcaf8905dcde0537806fb16b9ff4c-1 {}] regionserver.HStore(327): Store=2a8fcaf8905dcde0537806fb16b9ff4c/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,473 INFO [StoreOpener-2a8fcaf8905dcde0537806fb16b9ff4c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 2a8fcaf8905dcde0537806fb16b9ff4c 2024-11-18T18:24:21,475 INFO [StoreOpener-2a8fcaf8905dcde0537806fb16b9ff4c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2a8fcaf8905dcde0537806fb16b9ff4c columnFamilyName colfamily21 2024-11-18T18:24:21,475 DEBUG [StoreOpener-2a8fcaf8905dcde0537806fb16b9ff4c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,476 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731954260238.012c277c99c564dd90e8e5f6850b6ce0. 2024-11-18T18:24:21,476 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731954260238.012c277c99c564dd90e8e5f6850b6ce0. 2024-11-18T18:24:21,476 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731954260238.bb099f0975ebdb618ef8e3b2c263bbca. 2024-11-18T18:24:21,476 INFO [StoreOpener-2a8fcaf8905dcde0537806fb16b9ff4c-1 {}] regionserver.HStore(327): Store=2a8fcaf8905dcde0537806fb16b9ff4c/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,477 INFO [StoreOpener-2a8fcaf8905dcde0537806fb16b9ff4c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 2a8fcaf8905dcde0537806fb16b9ff4c 2024-11-18T18:24:21,477 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => bb099f0975ebdb618ef8e3b2c263bbca, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731954260238.bb099f0975ebdb618ef8e3b2c263bbca.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-18T18:24:21,477 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=012c277c99c564dd90e8e5f6850b6ce0, regionState=OPEN, openSeqNum=2, regionLocation=961a81da9827,42895,1731954257867 2024-11-18T18:24:21,477 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true bb099f0975ebdb618ef8e3b2c263bbca 2024-11-18T18:24:21,477 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731954260238.bb099f0975ebdb618ef8e3b2c263bbca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:21,477 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for bb099f0975ebdb618ef8e3b2c263bbca 2024-11-18T18:24:21,478 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for bb099f0975ebdb618ef8e3b2c263bbca 2024-11-18T18:24:21,479 INFO [StoreOpener-2a8fcaf8905dcde0537806fb16b9ff4c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2a8fcaf8905dcde0537806fb16b9ff4c columnFamilyName colfamily31 2024-11-18T18:24:21,479 DEBUG [StoreOpener-2a8fcaf8905dcde0537806fb16b9ff4c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,480 INFO [StoreOpener-2a8fcaf8905dcde0537806fb16b9ff4c-1 {}] regionserver.HStore(327): Store=2a8fcaf8905dcde0537806fb16b9ff4c/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,480 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for 2a8fcaf8905dcde0537806fb16b9ff4c 2024-11-18T18:24:21,481 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 012c277c99c564dd90e8e5f6850b6ce0, server=961a81da9827,42895,1731954257867 because future has completed 2024-11-18T18:24:21,481 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2a8fcaf8905dcde0537806fb16b9ff4c 2024-11-18T18:24:21,481 INFO [StoreOpener-bb099f0975ebdb618ef8e3b2c263bbca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region bb099f0975ebdb618ef8e3b2c263bbca 2024-11-18T18:24:21,482 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2a8fcaf8905dcde0537806fb16b9ff4c 2024-11-18T18:24:21,484 INFO [StoreOpener-bb099f0975ebdb618ef8e3b2c263bbca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bb099f0975ebdb618ef8e3b2c263bbca columnFamilyName colfamily11 2024-11-18T18:24:21,484 DEBUG [StoreOpener-bb099f0975ebdb618ef8e3b2c263bbca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,484 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for 2a8fcaf8905dcde0537806fb16b9ff4c 2024-11-18T18:24:21,484 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for 2a8fcaf8905dcde0537806fb16b9ff4c 2024-11-18T18:24:21,484 INFO [StoreOpener-bb099f0975ebdb618ef8e3b2c263bbca-1 {}] regionserver.HStore(327): Store=bb099f0975ebdb618ef8e3b2c263bbca/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,485 INFO [StoreOpener-bb099f0975ebdb618ef8e3b2c263bbca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region bb099f0975ebdb618ef8e3b2c263bbca 2024-11-18T18:24:21,485 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T18:24:21,486 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=18 2024-11-18T18:24:21,486 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure 012c277c99c564dd90e8e5f6850b6ce0, server=961a81da9827,42895,1731954257867 in 201 msec 2024-11-18T18:24:21,487 INFO [StoreOpener-bb099f0975ebdb618ef8e3b2c263bbca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bb099f0975ebdb618ef8e3b2c263bbca columnFamilyName colfamily21 2024-11-18T18:24:21,487 DEBUG [StoreOpener-bb099f0975ebdb618ef8e3b2c263bbca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,487 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for 2a8fcaf8905dcde0537806fb16b9ff4c 2024-11-18T18:24:21,488 INFO [StoreOpener-bb099f0975ebdb618ef8e3b2c263bbca-1 {}] regionserver.HStore(327): Store=bb099f0975ebdb618ef8e3b2c263bbca/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,488 INFO [StoreOpener-bb099f0975ebdb618ef8e3b2c263bbca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region bb099f0975ebdb618ef8e3b2c263bbca 2024-11-18T18:24:21,489 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=012c277c99c564dd90e8e5f6850b6ce0, ASSIGN in 420 msec 2024-11-18T18:24:21,490 INFO [StoreOpener-bb099f0975ebdb618ef8e3b2c263bbca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bb099f0975ebdb618ef8e3b2c263bbca columnFamilyName colfamily31 2024-11-18T18:24:21,490 DEBUG [StoreOpener-bb099f0975ebdb618ef8e3b2c263bbca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,491 INFO [StoreOpener-bb099f0975ebdb618ef8e3b2c263bbca-1 {}] regionserver.HStore(327): Store=bb099f0975ebdb618ef8e3b2c263bbca/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,491 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for bb099f0975ebdb618ef8e3b2c263bbca 2024-11-18T18:24:21,492 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bb099f0975ebdb618ef8e3b2c263bbca 2024-11-18T18:24:21,493 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bb099f0975ebdb618ef8e3b2c263bbca 2024-11-18T18:24:21,494 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for bb099f0975ebdb618ef8e3b2c263bbca 2024-11-18T18:24:21,494 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for bb099f0975ebdb618ef8e3b2c263bbca 2024-11-18T18:24:21,495 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T18:24:21,496 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for bb099f0975ebdb618ef8e3b2c263bbca 2024-11-18T18:24:21,498 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2a8fcaf8905dcde0537806fb16b9ff4c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T18:24:21,499 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bb099f0975ebdb618ef8e3b2c263bbca/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T18:24:21,499 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened 2a8fcaf8905dcde0537806fb16b9ff4c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73586547, jitterRate=0.09652499854564667}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T18:24:21,499 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2a8fcaf8905dcde0537806fb16b9ff4c 2024-11-18T18:24:21,499 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for 2a8fcaf8905dcde0537806fb16b9ff4c: Running coprocessor pre-open hook at 1731954261468Writing region info on filesystem at 1731954261468Initializing all the Stores at 1731954261469 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261469Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261469Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261469Cleaning up temporary data from old regions at 1731954261485 (+16 ms)Running coprocessor post-open hooks at 1731954261499 (+14 ms)Region opened successfully at 1731954261499 2024-11-18T18:24:21,499 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened bb099f0975ebdb618ef8e3b2c263bbca; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64336967, jitterRate=-0.04130448400974274}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T18:24:21,499 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bb099f0975ebdb618ef8e3b2c263bbca 2024-11-18T18:24:21,500 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for bb099f0975ebdb618ef8e3b2c263bbca: Running coprocessor pre-open hook at 1731954261478Writing region info on filesystem at 1731954261478Initializing all the Stores at 1731954261479 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261479Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261481 (+2 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261481Cleaning up temporary data from old regions at 1731954261494 (+13 ms)Running coprocessor post-open hooks at 1731954261499 (+5 ms)Region opened successfully at 1731954261500 (+1 ms) 2024-11-18T18:24:21,500 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731954260238.2a8fcaf8905dcde0537806fb16b9ff4c., pid=44, masterSystemTime=1731954261458 2024-11-18T18:24:21,501 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731954260238.bb099f0975ebdb618ef8e3b2c263bbca., pid=35, masterSystemTime=1731954261396 2024-11-18T18:24:21,504 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731954260238.2a8fcaf8905dcde0537806fb16b9ff4c. 2024-11-18T18:24:21,504 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731954260238.2a8fcaf8905dcde0537806fb16b9ff4c. 2024-11-18T18:24:21,505 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731954260238.f39571a93e25091090ea0831d54936b5. 2024-11-18T18:24:21,505 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => f39571a93e25091090ea0831d54936b5, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731954260238.f39571a93e25091090ea0831d54936b5.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-18T18:24:21,505 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true f39571a93e25091090ea0831d54936b5 2024-11-18T18:24:21,505 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=2a8fcaf8905dcde0537806fb16b9ff4c, regionState=OPEN, openSeqNum=2, regionLocation=961a81da9827,38709,1731954257723 2024-11-18T18:24:21,505 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731954260238.f39571a93e25091090ea0831d54936b5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:21,506 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for f39571a93e25091090ea0831d54936b5 2024-11-18T18:24:21,506 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for f39571a93e25091090ea0831d54936b5 2024-11-18T18:24:21,507 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731954260238.bb099f0975ebdb618ef8e3b2c263bbca. 2024-11-18T18:24:21,507 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731954260238.bb099f0975ebdb618ef8e3b2c263bbca. 2024-11-18T18:24:21,507 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731954260238.75a13ed51395b1016b90c6b0d90a7d0a. 2024-11-18T18:24:21,507 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => 75a13ed51395b1016b90c6b0d90a7d0a, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731954260238.75a13ed51395b1016b90c6b0d90a7d0a.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-18T18:24:21,508 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 75a13ed51395b1016b90c6b0d90a7d0a 2024-11-18T18:24:21,509 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731954260238.75a13ed51395b1016b90c6b0d90a7d0a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:21,509 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for 75a13ed51395b1016b90c6b0d90a7d0a 2024-11-18T18:24:21,509 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for 75a13ed51395b1016b90c6b0d90a7d0a 2024-11-18T18:24:21,509 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2a8fcaf8905dcde0537806fb16b9ff4c, server=961a81da9827,38709,1731954257723 because future has completed 2024-11-18T18:24:21,510 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=bb099f0975ebdb618ef8e3b2c263bbca, regionState=OPEN, openSeqNum=2, regionLocation=961a81da9827,42895,1731954257867 2024-11-18T18:24:21,512 INFO [StoreOpener-f39571a93e25091090ea0831d54936b5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region f39571a93e25091090ea0831d54936b5 2024-11-18T18:24:21,514 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure bb099f0975ebdb618ef8e3b2c263bbca, server=961a81da9827,42895,1731954257867 because future has completed 2024-11-18T18:24:21,515 INFO [StoreOpener-f39571a93e25091090ea0831d54936b5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f39571a93e25091090ea0831d54936b5 columnFamilyName colfamily11 2024-11-18T18:24:21,516 DEBUG [StoreOpener-f39571a93e25091090ea0831d54936b5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,516 INFO [StoreOpener-75a13ed51395b1016b90c6b0d90a7d0a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 75a13ed51395b1016b90c6b0d90a7d0a 2024-11-18T18:24:21,517 INFO [StoreOpener-f39571a93e25091090ea0831d54936b5-1 {}] regionserver.HStore(327): Store=f39571a93e25091090ea0831d54936b5/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,517 INFO [StoreOpener-f39571a93e25091090ea0831d54936b5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region f39571a93e25091090ea0831d54936b5 2024-11-18T18:24:21,519 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=27 2024-11-18T18:24:21,519 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 2a8fcaf8905dcde0537806fb16b9ff4c, server=961a81da9827,38709,1731954257723 in 189 msec 2024-11-18T18:24:21,519 INFO [StoreOpener-75a13ed51395b1016b90c6b0d90a7d0a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 75a13ed51395b1016b90c6b0d90a7d0a columnFamilyName colfamily11 2024-11-18T18:24:21,519 INFO [StoreOpener-f39571a93e25091090ea0831d54936b5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f39571a93e25091090ea0831d54936b5 columnFamilyName colfamily21 2024-11-18T18:24:21,519 DEBUG [StoreOpener-75a13ed51395b1016b90c6b0d90a7d0a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,519 DEBUG [StoreOpener-f39571a93e25091090ea0831d54936b5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,520 INFO [StoreOpener-75a13ed51395b1016b90c6b0d90a7d0a-1 {}] regionserver.HStore(327): Store=75a13ed51395b1016b90c6b0d90a7d0a/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,520 INFO [StoreOpener-75a13ed51395b1016b90c6b0d90a7d0a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 75a13ed51395b1016b90c6b0d90a7d0a 2024-11-18T18:24:21,520 INFO [StoreOpener-f39571a93e25091090ea0831d54936b5-1 {}] regionserver.HStore(327): Store=f39571a93e25091090ea0831d54936b5/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,521 INFO [StoreOpener-f39571a93e25091090ea0831d54936b5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region f39571a93e25091090ea0831d54936b5 2024-11-18T18:24:21,522 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2a8fcaf8905dcde0537806fb16b9ff4c, ASSIGN in 453 msec 2024-11-18T18:24:21,522 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731954260238.779798cd60db955ed78a4329cac08969. 2024-11-18T18:24:21,522 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => 779798cd60db955ed78a4329cac08969, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731954260238.779798cd60db955ed78a4329cac08969.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-18T18:24:21,522 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 779798cd60db955ed78a4329cac08969 2024-11-18T18:24:21,523 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731954260238.779798cd60db955ed78a4329cac08969.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:21,523 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for 779798cd60db955ed78a4329cac08969 2024-11-18T18:24:21,523 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for 779798cd60db955ed78a4329cac08969 2024-11-18T18:24:21,524 INFO [StoreOpener-75a13ed51395b1016b90c6b0d90a7d0a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 75a13ed51395b1016b90c6b0d90a7d0a columnFamilyName colfamily21 2024-11-18T18:24:21,524 INFO [StoreOpener-f39571a93e25091090ea0831d54936b5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f39571a93e25091090ea0831d54936b5 columnFamilyName colfamily31 2024-11-18T18:24:21,524 DEBUG [StoreOpener-75a13ed51395b1016b90c6b0d90a7d0a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,524 DEBUG [StoreOpener-f39571a93e25091090ea0831d54936b5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,526 INFO [StoreOpener-f39571a93e25091090ea0831d54936b5-1 {}] regionserver.HStore(327): Store=f39571a93e25091090ea0831d54936b5/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,526 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for f39571a93e25091090ea0831d54936b5 2024-11-18T18:24:21,526 INFO [StoreOpener-75a13ed51395b1016b90c6b0d90a7d0a-1 {}] regionserver.HStore(327): Store=75a13ed51395b1016b90c6b0d90a7d0a/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,526 INFO [StoreOpener-75a13ed51395b1016b90c6b0d90a7d0a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 75a13ed51395b1016b90c6b0d90a7d0a 2024-11-18T18:24:21,527 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f39571a93e25091090ea0831d54936b5 2024-11-18T18:24:21,528 INFO [StoreOpener-779798cd60db955ed78a4329cac08969-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 779798cd60db955ed78a4329cac08969 2024-11-18T18:24:21,528 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=8 2024-11-18T18:24:21,528 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure bb099f0975ebdb618ef8e3b2c263bbca, server=961a81da9827,42895,1731954257867 in 253 msec 2024-11-18T18:24:21,528 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f39571a93e25091090ea0831d54936b5 2024-11-18T18:24:21,529 INFO [StoreOpener-75a13ed51395b1016b90c6b0d90a7d0a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 75a13ed51395b1016b90c6b0d90a7d0a columnFamilyName colfamily31 2024-11-18T18:24:21,529 DEBUG [StoreOpener-75a13ed51395b1016b90c6b0d90a7d0a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,530 INFO [StoreOpener-75a13ed51395b1016b90c6b0d90a7d0a-1 {}] regionserver.HStore(327): Store=75a13ed51395b1016b90c6b0d90a7d0a/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,530 INFO [StoreOpener-779798cd60db955ed78a4329cac08969-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 779798cd60db955ed78a4329cac08969 columnFamilyName colfamily11 2024-11-18T18:24:21,530 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for 75a13ed51395b1016b90c6b0d90a7d0a 2024-11-18T18:24:21,530 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for f39571a93e25091090ea0831d54936b5 2024-11-18T18:24:21,530 DEBUG [StoreOpener-779798cd60db955ed78a4329cac08969-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,531 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for f39571a93e25091090ea0831d54936b5 2024-11-18T18:24:21,531 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=bb099f0975ebdb618ef8e3b2c263bbca, ASSIGN in 462 msec 2024-11-18T18:24:21,531 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T18:24:21,532 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/75a13ed51395b1016b90c6b0d90a7d0a 2024-11-18T18:24:21,532 INFO [StoreOpener-779798cd60db955ed78a4329cac08969-1 {}] regionserver.HStore(327): Store=779798cd60db955ed78a4329cac08969/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,532 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/75a13ed51395b1016b90c6b0d90a7d0a 2024-11-18T18:24:21,532 INFO [StoreOpener-779798cd60db955ed78a4329cac08969-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 779798cd60db955ed78a4329cac08969 2024-11-18T18:24:21,534 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for f39571a93e25091090ea0831d54936b5 2024-11-18T18:24:21,535 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for 75a13ed51395b1016b90c6b0d90a7d0a 2024-11-18T18:24:21,535 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for 75a13ed51395b1016b90c6b0d90a7d0a 2024-11-18T18:24:21,535 INFO [StoreOpener-779798cd60db955ed78a4329cac08969-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 779798cd60db955ed78a4329cac08969 columnFamilyName colfamily21 2024-11-18T18:24:21,535 DEBUG [StoreOpener-779798cd60db955ed78a4329cac08969-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,536 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T18:24:21,537 INFO [StoreOpener-779798cd60db955ed78a4329cac08969-1 {}] regionserver.HStore(327): Store=779798cd60db955ed78a4329cac08969/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,537 INFO [StoreOpener-779798cd60db955ed78a4329cac08969-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 779798cd60db955ed78a4329cac08969 2024-11-18T18:24:21,537 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f39571a93e25091090ea0831d54936b5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T18:24:21,538 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for 75a13ed51395b1016b90c6b0d90a7d0a 2024-11-18T18:24:21,538 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened f39571a93e25091090ea0831d54936b5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63679806, jitterRate=-0.051096946001052856}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T18:24:21,538 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f39571a93e25091090ea0831d54936b5 2024-11-18T18:24:21,538 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for f39571a93e25091090ea0831d54936b5: Running coprocessor pre-open hook at 1731954261506Writing region info on filesystem at 1731954261506Initializing all the Stores at 1731954261509 (+3 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261509Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261512 (+3 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261512Cleaning up temporary data from old regions at 1731954261531 (+19 ms)Running coprocessor post-open hooks at 1731954261538 (+7 ms)Region opened successfully at 1731954261538 2024-11-18T18:24:21,542 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731954260238.f39571a93e25091090ea0831d54936b5., pid=45, masterSystemTime=1731954261458 2024-11-18T18:24:21,542 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/75a13ed51395b1016b90c6b0d90a7d0a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T18:24:21,543 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened 75a13ed51395b1016b90c6b0d90a7d0a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59274075, jitterRate=-0.11674745380878448}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T18:24:21,543 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 75a13ed51395b1016b90c6b0d90a7d0a 2024-11-18T18:24:21,543 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for 75a13ed51395b1016b90c6b0d90a7d0a: Running coprocessor pre-open hook at 1731954261509Writing region info on filesystem at 1731954261509Initializing all the Stores at 1731954261511 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261511Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261515 (+4 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261515Cleaning up temporary data from old regions at 1731954261535 (+20 ms)Running coprocessor post-open hooks at 1731954261543 (+8 ms)Region opened successfully at 1731954261543 2024-11-18T18:24:21,545 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731954260238.75a13ed51395b1016b90c6b0d90a7d0a., pid=38, masterSystemTime=1731954261396 2024-11-18T18:24:21,545 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731954260238.f39571a93e25091090ea0831d54936b5. 2024-11-18T18:24:21,545 INFO [StoreOpener-779798cd60db955ed78a4329cac08969-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 779798cd60db955ed78a4329cac08969 columnFamilyName colfamily31 2024-11-18T18:24:21,545 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731954260238.f39571a93e25091090ea0831d54936b5. 2024-11-18T18:24:21,546 DEBUG [StoreOpener-779798cd60db955ed78a4329cac08969-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,546 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731954260238.fb91ff28ef3c18f0dc0d3f26aac68eca. 2024-11-18T18:24:21,546 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => fb91ff28ef3c18f0dc0d3f26aac68eca, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731954260238.fb91ff28ef3c18f0dc0d3f26aac68eca.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-18T18:24:21,546 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true fb91ff28ef3c18f0dc0d3f26aac68eca 2024-11-18T18:24:21,546 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731954260238.fb91ff28ef3c18f0dc0d3f26aac68eca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:21,546 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for fb91ff28ef3c18f0dc0d3f26aac68eca 2024-11-18T18:24:21,546 INFO [StoreOpener-779798cd60db955ed78a4329cac08969-1 {}] regionserver.HStore(327): Store=779798cd60db955ed78a4329cac08969/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,546 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for fb91ff28ef3c18f0dc0d3f26aac68eca 2024-11-18T18:24:21,547 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for 779798cd60db955ed78a4329cac08969 2024-11-18T18:24:21,548 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=f39571a93e25091090ea0831d54936b5, regionState=OPEN, openSeqNum=2, regionLocation=961a81da9827,38709,1731954257723 2024-11-18T18:24:21,549 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/779798cd60db955ed78a4329cac08969 2024-11-18T18:24:21,550 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/779798cd60db955ed78a4329cac08969 2024-11-18T18:24:21,552 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure f39571a93e25091090ea0831d54936b5, server=961a81da9827,38709,1731954257723 because future has completed 2024-11-18T18:24:21,552 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731954260238.75a13ed51395b1016b90c6b0d90a7d0a. 2024-11-18T18:24:21,552 INFO [StoreOpener-fb91ff28ef3c18f0dc0d3f26aac68eca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region fb91ff28ef3c18f0dc0d3f26aac68eca 2024-11-18T18:24:21,552 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731954260238.75a13ed51395b1016b90c6b0d90a7d0a. 2024-11-18T18:24:21,553 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731954260238.22971deb0db38b7cccde5c3bc0868f22. 2024-11-18T18:24:21,553 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 22971deb0db38b7cccde5c3bc0868f22, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731954260238.22971deb0db38b7cccde5c3bc0868f22.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-18T18:24:21,553 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=75a13ed51395b1016b90c6b0d90a7d0a, regionState=OPEN, openSeqNum=2, regionLocation=961a81da9827,42895,1731954257867 2024-11-18T18:24:21,554 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 22971deb0db38b7cccde5c3bc0868f22 2024-11-18T18:24:21,554 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731954260238.22971deb0db38b7cccde5c3bc0868f22.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:21,554 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 22971deb0db38b7cccde5c3bc0868f22 2024-11-18T18:24:21,554 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 22971deb0db38b7cccde5c3bc0868f22 2024-11-18T18:24:21,555 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for 779798cd60db955ed78a4329cac08969 2024-11-18T18:24:21,555 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for 779798cd60db955ed78a4329cac08969 2024-11-18T18:24:21,556 INFO [StoreOpener-fb91ff28ef3c18f0dc0d3f26aac68eca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fb91ff28ef3c18f0dc0d3f26aac68eca columnFamilyName colfamily11 2024-11-18T18:24:21,557 DEBUG [StoreOpener-fb91ff28ef3c18f0dc0d3f26aac68eca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,557 INFO [StoreOpener-22971deb0db38b7cccde5c3bc0868f22-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 22971deb0db38b7cccde5c3bc0868f22 2024-11-18T18:24:21,556 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 75a13ed51395b1016b90c6b0d90a7d0a, server=961a81da9827,42895,1731954257867 because future has completed 2024-11-18T18:24:21,558 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T18:24:21,558 INFO [StoreOpener-fb91ff28ef3c18f0dc0d3f26aac68eca-1 {}] regionserver.HStore(327): Store=fb91ff28ef3c18f0dc0d3f26aac68eca/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,558 INFO [StoreOpener-fb91ff28ef3c18f0dc0d3f26aac68eca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region fb91ff28ef3c18f0dc0d3f26aac68eca 2024-11-18T18:24:21,560 INFO [StoreOpener-22971deb0db38b7cccde5c3bc0868f22-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22971deb0db38b7cccde5c3bc0868f22 columnFamilyName colfamily11 2024-11-18T18:24:21,560 DEBUG [StoreOpener-22971deb0db38b7cccde5c3bc0868f22-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,561 INFO [StoreOpener-fb91ff28ef3c18f0dc0d3f26aac68eca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fb91ff28ef3c18f0dc0d3f26aac68eca columnFamilyName colfamily21 2024-11-18T18:24:21,561 DEBUG [StoreOpener-fb91ff28ef3c18f0dc0d3f26aac68eca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,561 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for 779798cd60db955ed78a4329cac08969 2024-11-18T18:24:21,561 INFO [StoreOpener-22971deb0db38b7cccde5c3bc0868f22-1 {}] regionserver.HStore(327): Store=22971deb0db38b7cccde5c3bc0868f22/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,561 INFO [StoreOpener-22971deb0db38b7cccde5c3bc0868f22-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 22971deb0db38b7cccde5c3bc0868f22 2024-11-18T18:24:21,562 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=13 2024-11-18T18:24:21,562 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure f39571a93e25091090ea0831d54936b5, server=961a81da9827,38709,1731954257723 in 216 msec 2024-11-18T18:24:21,562 INFO [StoreOpener-fb91ff28ef3c18f0dc0d3f26aac68eca-1 {}] regionserver.HStore(327): Store=fb91ff28ef3c18f0dc0d3f26aac68eca/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,562 INFO [StoreOpener-fb91ff28ef3c18f0dc0d3f26aac68eca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region fb91ff28ef3c18f0dc0d3f26aac68eca 2024-11-18T18:24:21,564 INFO [StoreOpener-22971deb0db38b7cccde5c3bc0868f22-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22971deb0db38b7cccde5c3bc0868f22 columnFamilyName colfamily21 2024-11-18T18:24:21,564 DEBUG [StoreOpener-22971deb0db38b7cccde5c3bc0868f22-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,565 INFO [StoreOpener-fb91ff28ef3c18f0dc0d3f26aac68eca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fb91ff28ef3c18f0dc0d3f26aac68eca columnFamilyName colfamily31 2024-11-18T18:24:21,565 DEBUG [StoreOpener-fb91ff28ef3c18f0dc0d3f26aac68eca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,565 INFO [StoreOpener-22971deb0db38b7cccde5c3bc0868f22-1 {}] regionserver.HStore(327): Store=22971deb0db38b7cccde5c3bc0868f22/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,565 INFO [StoreOpener-22971deb0db38b7cccde5c3bc0868f22-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 22971deb0db38b7cccde5c3bc0868f22 2024-11-18T18:24:21,566 INFO [StoreOpener-fb91ff28ef3c18f0dc0d3f26aac68eca-1 {}] regionserver.HStore(327): Store=fb91ff28ef3c18f0dc0d3f26aac68eca/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,566 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for fb91ff28ef3c18f0dc0d3f26aac68eca 2024-11-18T18:24:21,566 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=f39571a93e25091090ea0831d54936b5, ASSIGN in 496 msec 2024-11-18T18:24:21,566 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/779798cd60db955ed78a4329cac08969/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T18:24:21,567 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fb91ff28ef3c18f0dc0d3f26aac68eca 2024-11-18T18:24:21,568 INFO [StoreOpener-22971deb0db38b7cccde5c3bc0868f22-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22971deb0db38b7cccde5c3bc0868f22 columnFamilyName colfamily31 2024-11-18T18:24:21,568 DEBUG [StoreOpener-22971deb0db38b7cccde5c3bc0868f22-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,568 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fb91ff28ef3c18f0dc0d3f26aac68eca 2024-11-18T18:24:21,568 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened 779798cd60db955ed78a4329cac08969; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62854141, jitterRate=-0.06340031325817108}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T18:24:21,568 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 779798cd60db955ed78a4329cac08969 2024-11-18T18:24:21,568 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for 779798cd60db955ed78a4329cac08969: Running coprocessor pre-open hook at 1731954261523Writing region info on filesystem at 1731954261523Initializing all the Stores at 1731954261525 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261526 (+1 ms)Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261527 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261527Cleaning up temporary data from old regions at 1731954261555 (+28 ms)Running coprocessor post-open hooks at 1731954261568 (+13 ms)Region opened successfully at 1731954261568 2024-11-18T18:24:21,569 INFO [StoreOpener-22971deb0db38b7cccde5c3bc0868f22-1 {}] regionserver.HStore(327): Store=22971deb0db38b7cccde5c3bc0868f22/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,570 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 22971deb0db38b7cccde5c3bc0868f22 2024-11-18T18:24:21,571 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731954260238.779798cd60db955ed78a4329cac08969., pid=46, masterSystemTime=1731954261502 2024-11-18T18:24:21,572 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/22971deb0db38b7cccde5c3bc0868f22 2024-11-18T18:24:21,572 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for fb91ff28ef3c18f0dc0d3f26aac68eca 2024-11-18T18:24:21,572 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for fb91ff28ef3c18f0dc0d3f26aac68eca 2024-11-18T18:24:21,574 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=23 2024-11-18T18:24:21,574 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 75a13ed51395b1016b90c6b0d90a7d0a, server=961a81da9827,42895,1731954257867 in 280 msec 2024-11-18T18:24:21,574 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T18:24:21,574 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731954260238.779798cd60db955ed78a4329cac08969. 2024-11-18T18:24:21,575 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731954260238.779798cd60db955ed78a4329cac08969. 2024-11-18T18:24:21,575 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731954260238.81f0db1bea1e11879dc1cafb06c0756d. 2024-11-18T18:24:21,575 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/22971deb0db38b7cccde5c3bc0868f22 2024-11-18T18:24:21,576 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=779798cd60db955ed78a4329cac08969, regionState=OPEN, openSeqNum=2, regionLocation=961a81da9827,36379,1731954257826 2024-11-18T18:24:21,576 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => 81f0db1bea1e11879dc1cafb06c0756d, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731954260238.81f0db1bea1e11879dc1cafb06c0756d.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-18T18:24:21,577 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 81f0db1bea1e11879dc1cafb06c0756d 2024-11-18T18:24:21,577 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for fb91ff28ef3c18f0dc0d3f26aac68eca 2024-11-18T18:24:21,577 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731954260238.81f0db1bea1e11879dc1cafb06c0756d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:21,577 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for 81f0db1bea1e11879dc1cafb06c0756d 2024-11-18T18:24:21,578 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 22971deb0db38b7cccde5c3bc0868f22 2024-11-18T18:24:21,578 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 22971deb0db38b7cccde5c3bc0868f22 2024-11-18T18:24:21,579 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for 81f0db1bea1e11879dc1cafb06c0756d 2024-11-18T18:24:21,579 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T18:24:21,580 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 779798cd60db955ed78a4329cac08969, server=961a81da9827,36379,1731954257826 because future has completed 2024-11-18T18:24:21,582 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 22971deb0db38b7cccde5c3bc0868f22 2024-11-18T18:24:21,582 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fb91ff28ef3c18f0dc0d3f26aac68eca/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T18:24:21,582 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=75a13ed51395b1016b90c6b0d90a7d0a, ASSIGN in 512 msec 2024-11-18T18:24:21,583 INFO [StoreOpener-81f0db1bea1e11879dc1cafb06c0756d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 81f0db1bea1e11879dc1cafb06c0756d 2024-11-18T18:24:21,583 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened fb91ff28ef3c18f0dc0d3f26aac68eca; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65231453, jitterRate=-0.027975603938102722}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T18:24:21,583 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fb91ff28ef3c18f0dc0d3f26aac68eca 2024-11-18T18:24:21,584 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for fb91ff28ef3c18f0dc0d3f26aac68eca: Running coprocessor pre-open hook at 1731954261547Writing region info on filesystem at 1731954261547Initializing all the Stores at 1731954261551 (+4 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261551Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261552 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261552Cleaning up temporary data from old regions at 1731954261573 (+21 ms)Running coprocessor post-open hooks at 1731954261583 (+10 ms)Region opened successfully at 1731954261584 (+1 ms) 2024-11-18T18:24:21,585 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731954260238.fb91ff28ef3c18f0dc0d3f26aac68eca., pid=56, masterSystemTime=1731954261458 2024-11-18T18:24:21,586 INFO [StoreOpener-81f0db1bea1e11879dc1cafb06c0756d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81f0db1bea1e11879dc1cafb06c0756d columnFamilyName colfamily11 2024-11-18T18:24:21,586 DEBUG [StoreOpener-81f0db1bea1e11879dc1cafb06c0756d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,587 INFO [StoreOpener-81f0db1bea1e11879dc1cafb06c0756d-1 {}] regionserver.HStore(327): Store=81f0db1bea1e11879dc1cafb06c0756d/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,587 INFO [StoreOpener-81f0db1bea1e11879dc1cafb06c0756d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 81f0db1bea1e11879dc1cafb06c0756d 2024-11-18T18:24:21,588 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731954260238.fb91ff28ef3c18f0dc0d3f26aac68eca. 2024-11-18T18:24:21,588 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731954260238.fb91ff28ef3c18f0dc0d3f26aac68eca. 2024-11-18T18:24:21,588 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731954260238.08ccc32db22fb75f5466ad774ad9e8ba. 2024-11-18T18:24:21,588 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => 08ccc32db22fb75f5466ad774ad9e8ba, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731954260238.08ccc32db22fb75f5466ad774ad9e8ba.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-18T18:24:21,589 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/22971deb0db38b7cccde5c3bc0868f22/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T18:24:21,589 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 08ccc32db22fb75f5466ad774ad9e8ba 2024-11-18T18:24:21,589 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731954260238.08ccc32db22fb75f5466ad774ad9e8ba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:21,589 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=29 2024-11-18T18:24:21,589 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for 08ccc32db22fb75f5466ad774ad9e8ba 2024-11-18T18:24:21,589 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for 08ccc32db22fb75f5466ad774ad9e8ba 2024-11-18T18:24:21,589 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 779798cd60db955ed78a4329cac08969, server=961a81da9827,36379,1731954257826 in 236 msec 2024-11-18T18:24:21,590 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 22971deb0db38b7cccde5c3bc0868f22; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61726640, jitterRate=-0.08020138740539551}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T18:24:21,590 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 22971deb0db38b7cccde5c3bc0868f22 2024-11-18T18:24:21,590 INFO [StoreOpener-81f0db1bea1e11879dc1cafb06c0756d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81f0db1bea1e11879dc1cafb06c0756d columnFamilyName colfamily21 2024-11-18T18:24:21,590 DEBUG [StoreOpener-81f0db1bea1e11879dc1cafb06c0756d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,590 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 22971deb0db38b7cccde5c3bc0868f22: Running coprocessor pre-open hook at 1731954261554Writing region info on filesystem at 1731954261554Initializing all the Stores at 1731954261556 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261556Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261556Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261556Cleaning up temporary data from old regions at 1731954261578 (+22 ms)Running coprocessor post-open hooks at 1731954261590 (+12 ms)Region opened successfully at 1731954261590 2024-11-18T18:24:21,592 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731954260238.22971deb0db38b7cccde5c3bc0868f22., pid=34, masterSystemTime=1731954261396 2024-11-18T18:24:21,592 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=779798cd60db955ed78a4329cac08969, ASSIGN in 524 msec 2024-11-18T18:24:21,592 INFO [StoreOpener-81f0db1bea1e11879dc1cafb06c0756d-1 {}] regionserver.HStore(327): Store=81f0db1bea1e11879dc1cafb06c0756d/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,592 INFO [StoreOpener-81f0db1bea1e11879dc1cafb06c0756d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 81f0db1bea1e11879dc1cafb06c0756d 2024-11-18T18:24:21,593 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=fb91ff28ef3c18f0dc0d3f26aac68eca, regionState=OPEN, openSeqNum=2, regionLocation=961a81da9827,38709,1731954257723 2024-11-18T18:24:21,593 INFO [StoreOpener-08ccc32db22fb75f5466ad774ad9e8ba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 08ccc32db22fb75f5466ad774ad9e8ba 2024-11-18T18:24:21,595 INFO [StoreOpener-81f0db1bea1e11879dc1cafb06c0756d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81f0db1bea1e11879dc1cafb06c0756d columnFamilyName colfamily31 2024-11-18T18:24:21,595 DEBUG [StoreOpener-81f0db1bea1e11879dc1cafb06c0756d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,595 INFO [StoreOpener-81f0db1bea1e11879dc1cafb06c0756d-1 {}] regionserver.HStore(327): Store=81f0db1bea1e11879dc1cafb06c0756d/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,596 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731954260238.22971deb0db38b7cccde5c3bc0868f22. 2024-11-18T18:24:21,596 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731954260238.22971deb0db38b7cccde5c3bc0868f22. 2024-11-18T18:24:21,596 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731954260238.aaf2435418703b2a31c1190af1560ba8. 2024-11-18T18:24:21,596 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => aaf2435418703b2a31c1190af1560ba8, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731954260238.aaf2435418703b2a31c1190af1560ba8.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-18T18:24:21,597 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for 81f0db1bea1e11879dc1cafb06c0756d 2024-11-18T18:24:21,597 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true aaf2435418703b2a31c1190af1560ba8 2024-11-18T18:24:21,597 INFO [StoreOpener-08ccc32db22fb75f5466ad774ad9e8ba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 08ccc32db22fb75f5466ad774ad9e8ba columnFamilyName colfamily11 2024-11-18T18:24:21,597 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731954260238.aaf2435418703b2a31c1190af1560ba8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:21,597 DEBUG [StoreOpener-08ccc32db22fb75f5466ad774ad9e8ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,597 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for aaf2435418703b2a31c1190af1560ba8 2024-11-18T18:24:21,597 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for aaf2435418703b2a31c1190af1560ba8 2024-11-18T18:24:21,598 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=22971deb0db38b7cccde5c3bc0868f22, regionState=OPEN, openSeqNum=2, regionLocation=961a81da9827,42895,1731954257867 2024-11-18T18:24:21,598 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/81f0db1bea1e11879dc1cafb06c0756d 2024-11-18T18:24:21,599 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure fb91ff28ef3c18f0dc0d3f26aac68eca, server=961a81da9827,38709,1731954257723 because future has completed 2024-11-18T18:24:21,599 INFO [StoreOpener-08ccc32db22fb75f5466ad774ad9e8ba-1 {}] regionserver.HStore(327): Store=08ccc32db22fb75f5466ad774ad9e8ba/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,599 INFO [StoreOpener-08ccc32db22fb75f5466ad774ad9e8ba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 08ccc32db22fb75f5466ad774ad9e8ba 2024-11-18T18:24:21,600 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/81f0db1bea1e11879dc1cafb06c0756d 2024-11-18T18:24:21,602 INFO [StoreOpener-08ccc32db22fb75f5466ad774ad9e8ba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 08ccc32db22fb75f5466ad774ad9e8ba columnFamilyName colfamily21 2024-11-18T18:24:21,602 DEBUG [StoreOpener-08ccc32db22fb75f5466ad774ad9e8ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,602 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for 81f0db1bea1e11879dc1cafb06c0756d 2024-11-18T18:24:21,602 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for 81f0db1bea1e11879dc1cafb06c0756d 2024-11-18T18:24:21,602 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 22971deb0db38b7cccde5c3bc0868f22, server=961a81da9827,42895,1731954257867 because future has completed 2024-11-18T18:24:21,603 INFO [StoreOpener-08ccc32db22fb75f5466ad774ad9e8ba-1 {}] regionserver.HStore(327): Store=08ccc32db22fb75f5466ad774ad9e8ba/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,603 INFO [StoreOpener-08ccc32db22fb75f5466ad774ad9e8ba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 08ccc32db22fb75f5466ad774ad9e8ba 2024-11-18T18:24:21,606 INFO [StoreOpener-aaf2435418703b2a31c1190af1560ba8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region aaf2435418703b2a31c1190af1560ba8 2024-11-18T18:24:21,606 INFO [StoreOpener-08ccc32db22fb75f5466ad774ad9e8ba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 08ccc32db22fb75f5466ad774ad9e8ba columnFamilyName colfamily31 2024-11-18T18:24:21,606 DEBUG [StoreOpener-08ccc32db22fb75f5466ad774ad9e8ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,606 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T18:24:21,607 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=10 2024-11-18T18:24:21,607 INFO [StoreOpener-08ccc32db22fb75f5466ad774ad9e8ba-1 {}] regionserver.HStore(327): Store=08ccc32db22fb75f5466ad774ad9e8ba/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,607 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for 08ccc32db22fb75f5466ad774ad9e8ba 2024-11-18T18:24:21,607 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure fb91ff28ef3c18f0dc0d3f26aac68eca, server=961a81da9827,38709,1731954257723 in 204 msec 2024-11-18T18:24:21,608 INFO [StoreOpener-aaf2435418703b2a31c1190af1560ba8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aaf2435418703b2a31c1190af1560ba8 columnFamilyName colfamily11 2024-11-18T18:24:21,608 DEBUG [StoreOpener-aaf2435418703b2a31c1190af1560ba8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,609 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/08ccc32db22fb75f5466ad774ad9e8ba 2024-11-18T18:24:21,609 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for 81f0db1bea1e11879dc1cafb06c0756d 2024-11-18T18:24:21,609 INFO [StoreOpener-aaf2435418703b2a31c1190af1560ba8-1 {}] regionserver.HStore(327): Store=aaf2435418703b2a31c1190af1560ba8/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,609 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/08ccc32db22fb75f5466ad774ad9e8ba 2024-11-18T18:24:21,609 INFO [StoreOpener-aaf2435418703b2a31c1190af1560ba8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region aaf2435418703b2a31c1190af1560ba8 2024-11-18T18:24:21,612 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for 08ccc32db22fb75f5466ad774ad9e8ba 2024-11-18T18:24:21,612 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for 08ccc32db22fb75f5466ad774ad9e8ba 2024-11-18T18:24:21,612 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fb91ff28ef3c18f0dc0d3f26aac68eca, ASSIGN in 542 msec 2024-11-18T18:24:21,612 INFO [StoreOpener-aaf2435418703b2a31c1190af1560ba8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aaf2435418703b2a31c1190af1560ba8 columnFamilyName colfamily21 2024-11-18T18:24:21,612 DEBUG [StoreOpener-aaf2435418703b2a31c1190af1560ba8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,613 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T18:24:21,613 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/81f0db1bea1e11879dc1cafb06c0756d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T18:24:21,615 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened 81f0db1bea1e11879dc1cafb06c0756d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60320010, jitterRate=-0.10116180777549744}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T18:24:21,615 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 81f0db1bea1e11879dc1cafb06c0756d 2024-11-18T18:24:21,615 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for 81f0db1bea1e11879dc1cafb06c0756d: Running coprocessor pre-open hook at 1731954261579Writing region info on filesystem at 1731954261579Initializing all the Stores at 1731954261582 (+3 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261582Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261583 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261583Cleaning up temporary data from old regions at 1731954261602 (+19 ms)Running coprocessor post-open hooks at 1731954261615 (+13 ms)Region opened successfully at 1731954261615 2024-11-18T18:24:21,616 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for 08ccc32db22fb75f5466ad774ad9e8ba 2024-11-18T18:24:21,615 INFO [StoreOpener-aaf2435418703b2a31c1190af1560ba8-1 {}] regionserver.HStore(327): Store=aaf2435418703b2a31c1190af1560ba8/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,616 INFO [StoreOpener-aaf2435418703b2a31c1190af1560ba8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region aaf2435418703b2a31c1190af1560ba8 2024-11-18T18:24:21,617 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731954260238.81f0db1bea1e11879dc1cafb06c0756d., pid=54, masterSystemTime=1731954261502 2024-11-18T18:24:21,619 INFO [StoreOpener-aaf2435418703b2a31c1190af1560ba8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aaf2435418703b2a31c1190af1560ba8 columnFamilyName colfamily31 2024-11-18T18:24:21,619 DEBUG [StoreOpener-aaf2435418703b2a31c1190af1560ba8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,619 INFO [StoreOpener-aaf2435418703b2a31c1190af1560ba8-1 {}] regionserver.HStore(327): Store=aaf2435418703b2a31c1190af1560ba8/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,619 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for aaf2435418703b2a31c1190af1560ba8 2024-11-18T18:24:21,620 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=6 2024-11-18T18:24:21,620 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure 22971deb0db38b7cccde5c3bc0868f22, server=961a81da9827,42895,1731954257867 in 348 msec 2024-11-18T18:24:21,620 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731954260238.81f0db1bea1e11879dc1cafb06c0756d. 2024-11-18T18:24:21,620 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731954260238.81f0db1bea1e11879dc1cafb06c0756d. 2024-11-18T18:24:21,620 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aaf2435418703b2a31c1190af1560ba8 2024-11-18T18:24:21,621 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731954260238.43138a6a12c2efea7093064622bb3cd8. 2024-11-18T18:24:21,621 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aaf2435418703b2a31c1190af1560ba8 2024-11-18T18:24:21,621 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=81f0db1bea1e11879dc1cafb06c0756d, regionState=OPEN, openSeqNum=2, regionLocation=961a81da9827,36379,1731954257826 2024-11-18T18:24:21,621 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => 43138a6a12c2efea7093064622bb3cd8, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731954260238.43138a6a12c2efea7093064622bb3cd8.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-18T18:24:21,622 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 43138a6a12c2efea7093064622bb3cd8 2024-11-18T18:24:21,622 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731954260238.43138a6a12c2efea7093064622bb3cd8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:21,622 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for 43138a6a12c2efea7093064622bb3cd8 2024-11-18T18:24:21,622 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for 43138a6a12c2efea7093064622bb3cd8 2024-11-18T18:24:21,623 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for aaf2435418703b2a31c1190af1560ba8 2024-11-18T18:24:21,623 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for aaf2435418703b2a31c1190af1560ba8 2024-11-18T18:24:21,624 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T18:24:21,625 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 81f0db1bea1e11879dc1cafb06c0756d, server=961a81da9827,36379,1731954257826 because future has completed 2024-11-18T18:24:21,626 INFO [StoreOpener-43138a6a12c2efea7093064622bb3cd8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 43138a6a12c2efea7093064622bb3cd8 2024-11-18T18:24:21,626 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for aaf2435418703b2a31c1190af1560ba8 2024-11-18T18:24:21,628 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=22971deb0db38b7cccde5c3bc0868f22, ASSIGN in 558 msec 2024-11-18T18:24:21,630 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aaf2435418703b2a31c1190af1560ba8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T18:24:21,631 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened aaf2435418703b2a31c1190af1560ba8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60510746, jitterRate=-0.09831961989402771}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T18:24:21,631 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for aaf2435418703b2a31c1190af1560ba8 2024-11-18T18:24:21,631 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for aaf2435418703b2a31c1190af1560ba8: Running coprocessor pre-open hook at 1731954261597Writing region info on filesystem at 1731954261597Initializing all the Stores at 1731954261600 (+3 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261600Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261605 (+5 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261605Cleaning up temporary data from old regions at 1731954261623 (+18 ms)Running coprocessor post-open hooks at 1731954261631 (+8 ms)Region opened successfully at 1731954261631 2024-11-18T18:24:21,632 INFO [StoreOpener-43138a6a12c2efea7093064622bb3cd8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 43138a6a12c2efea7093064622bb3cd8 columnFamilyName colfamily11 2024-11-18T18:24:21,632 DEBUG [StoreOpener-43138a6a12c2efea7093064622bb3cd8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,633 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731954260238.aaf2435418703b2a31c1190af1560ba8., pid=39, masterSystemTime=1731954261396 2024-11-18T18:24:21,633 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/08ccc32db22fb75f5466ad774ad9e8ba/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T18:24:21,633 INFO [StoreOpener-43138a6a12c2efea7093064622bb3cd8-1 {}] regionserver.HStore(327): Store=43138a6a12c2efea7093064622bb3cd8/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,633 INFO [StoreOpener-43138a6a12c2efea7093064622bb3cd8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 43138a6a12c2efea7093064622bb3cd8 2024-11-18T18:24:21,633 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=26 2024-11-18T18:24:21,633 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 81f0db1bea1e11879dc1cafb06c0756d, server=961a81da9827,36379,1731954257826 in 240 msec 2024-11-18T18:24:21,634 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened 08ccc32db22fb75f5466ad774ad9e8ba; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66204682, jitterRate=-0.013473361730575562}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T18:24:21,634 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 08ccc32db22fb75f5466ad774ad9e8ba 2024-11-18T18:24:21,634 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for 08ccc32db22fb75f5466ad774ad9e8ba: Running coprocessor pre-open hook at 1731954261589Writing region info on filesystem at 1731954261589Initializing all the Stores at 1731954261591 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261591Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261593 (+2 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261593Cleaning up temporary data from old regions at 1731954261612 (+19 ms)Running coprocessor post-open hooks at 1731954261634 (+22 ms)Region opened successfully at 1731954261634 2024-11-18T18:24:21,638 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731954260238.08ccc32db22fb75f5466ad774ad9e8ba., pid=40, masterSystemTime=1731954261458 2024-11-18T18:24:21,638 INFO [StoreOpener-43138a6a12c2efea7093064622bb3cd8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 43138a6a12c2efea7093064622bb3cd8 columnFamilyName colfamily21 2024-11-18T18:24:21,640 DEBUG [StoreOpener-43138a6a12c2efea7093064622bb3cd8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,640 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731954260238.aaf2435418703b2a31c1190af1560ba8. 2024-11-18T18:24:21,640 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731954260238.aaf2435418703b2a31c1190af1560ba8. 2024-11-18T18:24:21,640 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731954260238.c8b4dd2a6d8dd59085c547b55aac0de6. 2024-11-18T18:24:21,640 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=81f0db1bea1e11879dc1cafb06c0756d, ASSIGN in 567 msec 2024-11-18T18:24:21,640 INFO [StoreOpener-43138a6a12c2efea7093064622bb3cd8-1 {}] regionserver.HStore(327): Store=43138a6a12c2efea7093064622bb3cd8/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,640 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => c8b4dd2a6d8dd59085c547b55aac0de6, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731954260238.c8b4dd2a6d8dd59085c547b55aac0de6.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-18T18:24:21,641 INFO [StoreOpener-43138a6a12c2efea7093064622bb3cd8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 43138a6a12c2efea7093064622bb3cd8 2024-11-18T18:24:21,641 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731954260238.08ccc32db22fb75f5466ad774ad9e8ba. 2024-11-18T18:24:21,641 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731954260238.08ccc32db22fb75f5466ad774ad9e8ba. 2024-11-18T18:24:21,641 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true c8b4dd2a6d8dd59085c547b55aac0de6 2024-11-18T18:24:21,641 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731954260238.69ae79c6f9eeed559fa88701f773706c. 2024-11-18T18:24:21,641 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731954260238.c8b4dd2a6d8dd59085c547b55aac0de6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:21,641 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => 69ae79c6f9eeed559fa88701f773706c, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731954260238.69ae79c6f9eeed559fa88701f773706c.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-18T18:24:21,641 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for c8b4dd2a6d8dd59085c547b55aac0de6 2024-11-18T18:24:21,642 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for c8b4dd2a6d8dd59085c547b55aac0de6 2024-11-18T18:24:21,642 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 69ae79c6f9eeed559fa88701f773706c 2024-11-18T18:24:21,642 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731954260238.69ae79c6f9eeed559fa88701f773706c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:21,642 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for 69ae79c6f9eeed559fa88701f773706c 2024-11-18T18:24:21,642 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for 69ae79c6f9eeed559fa88701f773706c 2024-11-18T18:24:21,643 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=08ccc32db22fb75f5466ad774ad9e8ba, regionState=OPEN, openSeqNum=2, regionLocation=961a81da9827,38709,1731954257723 2024-11-18T18:24:21,644 INFO [StoreOpener-43138a6a12c2efea7093064622bb3cd8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 43138a6a12c2efea7093064622bb3cd8 columnFamilyName colfamily31 2024-11-18T18:24:21,644 DEBUG [StoreOpener-43138a6a12c2efea7093064622bb3cd8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,645 INFO [StoreOpener-43138a6a12c2efea7093064622bb3cd8-1 {}] regionserver.HStore(327): Store=43138a6a12c2efea7093064622bb3cd8/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,645 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for 43138a6a12c2efea7093064622bb3cd8 2024-11-18T18:24:21,647 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/43138a6a12c2efea7093064622bb3cd8 2024-11-18T18:24:21,647 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/43138a6a12c2efea7093064622bb3cd8 2024-11-18T18:24:21,648 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 08ccc32db22fb75f5466ad774ad9e8ba, server=961a81da9827,38709,1731954257723 because future has completed 2024-11-18T18:24:21,649 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=aaf2435418703b2a31c1190af1560ba8, regionState=OPEN, openSeqNum=2, regionLocation=961a81da9827,42895,1731954257867 2024-11-18T18:24:21,649 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for 43138a6a12c2efea7093064622bb3cd8 2024-11-18T18:24:21,649 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for 43138a6a12c2efea7093064622bb3cd8 2024-11-18T18:24:21,649 INFO [StoreOpener-69ae79c6f9eeed559fa88701f773706c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 69ae79c6f9eeed559fa88701f773706c 2024-11-18T18:24:21,652 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T18:24:21,654 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure aaf2435418703b2a31c1190af1560ba8, server=961a81da9827,42895,1731954257867 because future has completed 2024-11-18T18:24:21,654 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33089 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=961a81da9827,42895,1731954257867, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=aaf2435418703b2a31c1190af1560ba8. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-18T18:24:21,655 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for 43138a6a12c2efea7093064622bb3cd8 2024-11-18T18:24:21,655 INFO [StoreOpener-69ae79c6f9eeed559fa88701f773706c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 69ae79c6f9eeed559fa88701f773706c columnFamilyName colfamily11 2024-11-18T18:24:21,656 DEBUG [StoreOpener-69ae79c6f9eeed559fa88701f773706c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,656 INFO [StoreOpener-69ae79c6f9eeed559fa88701f773706c-1 {}] regionserver.HStore(327): Store=69ae79c6f9eeed559fa88701f773706c/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,656 INFO [StoreOpener-69ae79c6f9eeed559fa88701f773706c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 69ae79c6f9eeed559fa88701f773706c 2024-11-18T18:24:21,658 INFO [StoreOpener-c8b4dd2a6d8dd59085c547b55aac0de6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region c8b4dd2a6d8dd59085c547b55aac0de6 2024-11-18T18:24:21,660 INFO [StoreOpener-69ae79c6f9eeed559fa88701f773706c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 69ae79c6f9eeed559fa88701f773706c columnFamilyName colfamily21 2024-11-18T18:24:21,660 DEBUG [StoreOpener-69ae79c6f9eeed559fa88701f773706c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,660 INFO [StoreOpener-69ae79c6f9eeed559fa88701f773706c-1 {}] regionserver.HStore(327): Store=69ae79c6f9eeed559fa88701f773706c/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,660 INFO [StoreOpener-c8b4dd2a6d8dd59085c547b55aac0de6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c8b4dd2a6d8dd59085c547b55aac0de6 columnFamilyName colfamily11 2024-11-18T18:24:21,661 DEBUG [StoreOpener-c8b4dd2a6d8dd59085c547b55aac0de6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,661 INFO [StoreOpener-69ae79c6f9eeed559fa88701f773706c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 69ae79c6f9eeed559fa88701f773706c 2024-11-18T18:24:21,662 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=24 2024-11-18T18:24:21,662 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure aaf2435418703b2a31c1190af1560ba8, server=961a81da9827,42895,1731954257867 in 362 msec 2024-11-18T18:24:21,663 INFO [StoreOpener-69ae79c6f9eeed559fa88701f773706c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 69ae79c6f9eeed559fa88701f773706c columnFamilyName colfamily31 2024-11-18T18:24:21,663 DEBUG [StoreOpener-69ae79c6f9eeed559fa88701f773706c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,664 INFO [StoreOpener-69ae79c6f9eeed559fa88701f773706c-1 {}] regionserver.HStore(327): Store=69ae79c6f9eeed559fa88701f773706c/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,664 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for 69ae79c6f9eeed559fa88701f773706c 2024-11-18T18:24:21,665 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=aaf2435418703b2a31c1190af1560ba8, ASSIGN in 596 msec 2024-11-18T18:24:21,666 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/69ae79c6f9eeed559fa88701f773706c 2024-11-18T18:24:21,667 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/69ae79c6f9eeed559fa88701f773706c 2024-11-18T18:24:21,667 INFO [StoreOpener-c8b4dd2a6d8dd59085c547b55aac0de6-1 {}] regionserver.HStore(327): Store=c8b4dd2a6d8dd59085c547b55aac0de6/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,667 INFO [StoreOpener-c8b4dd2a6d8dd59085c547b55aac0de6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region c8b4dd2a6d8dd59085c547b55aac0de6 2024-11-18T18:24:21,669 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for 69ae79c6f9eeed559fa88701f773706c 2024-11-18T18:24:21,669 INFO [StoreOpener-c8b4dd2a6d8dd59085c547b55aac0de6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c8b4dd2a6d8dd59085c547b55aac0de6 columnFamilyName colfamily21 2024-11-18T18:24:21,669 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for 69ae79c6f9eeed559fa88701f773706c 2024-11-18T18:24:21,669 DEBUG [StoreOpener-c8b4dd2a6d8dd59085c547b55aac0de6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,670 INFO [StoreOpener-c8b4dd2a6d8dd59085c547b55aac0de6-1 {}] regionserver.HStore(327): Store=c8b4dd2a6d8dd59085c547b55aac0de6/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,670 INFO [StoreOpener-c8b4dd2a6d8dd59085c547b55aac0de6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region c8b4dd2a6d8dd59085c547b55aac0de6 2024-11-18T18:24:21,670 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T18:24:21,670 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=30 2024-11-18T18:24:21,670 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure 08ccc32db22fb75f5466ad774ad9e8ba, server=961a81da9827,38709,1731954257723 in 362 msec 2024-11-18T18:24:21,672 INFO [StoreOpener-c8b4dd2a6d8dd59085c547b55aac0de6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c8b4dd2a6d8dd59085c547b55aac0de6 columnFamilyName colfamily31 2024-11-18T18:24:21,672 DEBUG [StoreOpener-c8b4dd2a6d8dd59085c547b55aac0de6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,673 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for 69ae79c6f9eeed559fa88701f773706c 2024-11-18T18:24:21,673 INFO [StoreOpener-c8b4dd2a6d8dd59085c547b55aac0de6-1 {}] regionserver.HStore(327): Store=c8b4dd2a6d8dd59085c547b55aac0de6/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,673 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=08ccc32db22fb75f5466ad774ad9e8ba, ASSIGN in 605 msec 2024-11-18T18:24:21,673 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for c8b4dd2a6d8dd59085c547b55aac0de6 2024-11-18T18:24:21,674 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c8b4dd2a6d8dd59085c547b55aac0de6 2024-11-18T18:24:21,675 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c8b4dd2a6d8dd59085c547b55aac0de6 2024-11-18T18:24:21,677 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/43138a6a12c2efea7093064622bb3cd8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T18:24:21,677 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for c8b4dd2a6d8dd59085c547b55aac0de6 2024-11-18T18:24:21,677 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for c8b4dd2a6d8dd59085c547b55aac0de6 2024-11-18T18:24:21,678 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened 43138a6a12c2efea7093064622bb3cd8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71389545, jitterRate=0.06378711760044098}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T18:24:21,678 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 43138a6a12c2efea7093064622bb3cd8 2024-11-18T18:24:21,678 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T18:24:21,679 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for 43138a6a12c2efea7093064622bb3cd8: Running coprocessor pre-open hook at 1731954261623Writing region info on filesystem at 1731954261623Initializing all the Stores at 1731954261625 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261625Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261625Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261625Cleaning up temporary data from old regions at 1731954261649 (+24 ms)Running coprocessor post-open hooks at 1731954261678 (+29 ms)Region opened successfully at 1731954261678 2024-11-18T18:24:21,680 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for c8b4dd2a6d8dd59085c547b55aac0de6 2024-11-18T18:24:21,681 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731954260238.43138a6a12c2efea7093064622bb3cd8., pid=48, masterSystemTime=1731954261502 2024-11-18T18:24:21,684 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731954260238.43138a6a12c2efea7093064622bb3cd8. 2024-11-18T18:24:21,685 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731954260238.43138a6a12c2efea7093064622bb3cd8. 2024-11-18T18:24:21,685 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731954260238.d1ad74f79ba4d7fd1619c83a55106331. 2024-11-18T18:24:21,685 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => d1ad74f79ba4d7fd1619c83a55106331, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731954260238.d1ad74f79ba4d7fd1619c83a55106331.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-18T18:24:21,686 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true d1ad74f79ba4d7fd1619c83a55106331 2024-11-18T18:24:21,686 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731954260238.d1ad74f79ba4d7fd1619c83a55106331.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:21,686 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for d1ad74f79ba4d7fd1619c83a55106331 2024-11-18T18:24:21,686 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for d1ad74f79ba4d7fd1619c83a55106331 2024-11-18T18:24:21,686 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=43138a6a12c2efea7093064622bb3cd8, regionState=OPEN, openSeqNum=2, regionLocation=961a81da9827,36379,1731954257826 2024-11-18T18:24:21,690 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 43138a6a12c2efea7093064622bb3cd8, server=961a81da9827,36379,1731954257826 because future has completed 2024-11-18T18:24:21,695 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=7 2024-11-18T18:24:21,695 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 43138a6a12c2efea7093064622bb3cd8, server=961a81da9827,36379,1731954257826 in 330 msec 2024-11-18T18:24:21,695 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/69ae79c6f9eeed559fa88701f773706c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T18:24:21,696 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened 69ae79c6f9eeed559fa88701f773706c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61529482, jitterRate=-0.08313927054405212}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T18:24:21,697 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 69ae79c6f9eeed559fa88701f773706c 2024-11-18T18:24:21,697 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for 69ae79c6f9eeed559fa88701f773706c: Running coprocessor pre-open hook at 1731954261642Writing region info on filesystem at 1731954261642Initializing all the Stores at 1731954261644 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261644Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261646 (+2 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261646Cleaning up temporary data from old regions at 1731954261669 (+23 ms)Running coprocessor post-open hooks at 1731954261697 (+28 ms)Region opened successfully at 1731954261697 2024-11-18T18:24:21,698 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731954260238.69ae79c6f9eeed559fa88701f773706c., pid=43, masterSystemTime=1731954261458 2024-11-18T18:24:21,698 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=43138a6a12c2efea7093064622bb3cd8, ASSIGN in 630 msec 2024-11-18T18:24:21,702 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731954260238.69ae79c6f9eeed559fa88701f773706c. 2024-11-18T18:24:21,702 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731954260238.69ae79c6f9eeed559fa88701f773706c. 2024-11-18T18:24:21,702 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731954260238.140f5077d3b730b643b558ebe8a13abd. 2024-11-18T18:24:21,702 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => 140f5077d3b730b643b558ebe8a13abd, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731954260238.140f5077d3b730b643b558ebe8a13abd.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-18T18:24:21,703 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=69ae79c6f9eeed559fa88701f773706c, regionState=OPEN, openSeqNum=2, regionLocation=961a81da9827,38709,1731954257723 2024-11-18T18:24:21,703 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 140f5077d3b730b643b558ebe8a13abd 2024-11-18T18:24:21,703 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731954260238.140f5077d3b730b643b558ebe8a13abd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:21,703 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for 140f5077d3b730b643b558ebe8a13abd 2024-11-18T18:24:21,703 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for 140f5077d3b730b643b558ebe8a13abd 2024-11-18T18:24:21,707 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c8b4dd2a6d8dd59085c547b55aac0de6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T18:24:21,707 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 69ae79c6f9eeed559fa88701f773706c, server=961a81da9827,38709,1731954257723 because future has completed 2024-11-18T18:24:21,708 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened c8b4dd2a6d8dd59085c547b55aac0de6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68302689, jitterRate=0.017789378762245178}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T18:24:21,708 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c8b4dd2a6d8dd59085c547b55aac0de6 2024-11-18T18:24:21,708 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for c8b4dd2a6d8dd59085c547b55aac0de6: Running coprocessor pre-open hook at 1731954261642Writing region info on filesystem at 1731954261642Initializing all the Stores at 1731954261644 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261645 (+1 ms)Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261657 (+12 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261657Cleaning up temporary data from old regions at 1731954261677 (+20 ms)Running coprocessor post-open hooks at 1731954261708 (+31 ms)Region opened successfully at 1731954261708 2024-11-18T18:24:21,709 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731954260238.c8b4dd2a6d8dd59085c547b55aac0de6., pid=33, masterSystemTime=1731954261396 2024-11-18T18:24:21,713 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731954260238.c8b4dd2a6d8dd59085c547b55aac0de6. 2024-11-18T18:24:21,713 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731954260238.c8b4dd2a6d8dd59085c547b55aac0de6. 2024-11-18T18:24:21,713 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731954260238.579ee42a94597fdc1db9a4820a53ee64. 2024-11-18T18:24:21,713 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => 579ee42a94597fdc1db9a4820a53ee64, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731954260238.579ee42a94597fdc1db9a4820a53ee64.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-18T18:24:21,714 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 579ee42a94597fdc1db9a4820a53ee64 2024-11-18T18:24:21,714 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731954260238.579ee42a94597fdc1db9a4820a53ee64.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:21,714 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for 579ee42a94597fdc1db9a4820a53ee64 2024-11-18T18:24:21,714 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for 579ee42a94597fdc1db9a4820a53ee64 2024-11-18T18:24:21,714 INFO [StoreOpener-d1ad74f79ba4d7fd1619c83a55106331-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region d1ad74f79ba4d7fd1619c83a55106331 2024-11-18T18:24:21,718 INFO [StoreOpener-140f5077d3b730b643b558ebe8a13abd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 140f5077d3b730b643b558ebe8a13abd 2024-11-18T18:24:21,719 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=22 2024-11-18T18:24:21,719 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure 69ae79c6f9eeed559fa88701f773706c, server=961a81da9827,38709,1731954257723 in 392 msec 2024-11-18T18:24:21,721 INFO [StoreOpener-140f5077d3b730b643b558ebe8a13abd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 140f5077d3b730b643b558ebe8a13abd columnFamilyName colfamily11 2024-11-18T18:24:21,721 DEBUG [StoreOpener-140f5077d3b730b643b558ebe8a13abd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,722 INFO [StoreOpener-140f5077d3b730b643b558ebe8a13abd-1 {}] regionserver.HStore(327): Store=140f5077d3b730b643b558ebe8a13abd/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,722 INFO [StoreOpener-140f5077d3b730b643b558ebe8a13abd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 140f5077d3b730b643b558ebe8a13abd 2024-11-18T18:24:21,722 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=69ae79c6f9eeed559fa88701f773706c, ASSIGN in 653 msec 2024-11-18T18:24:21,723 INFO [StoreOpener-d1ad74f79ba4d7fd1619c83a55106331-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1ad74f79ba4d7fd1619c83a55106331 columnFamilyName colfamily11 2024-11-18T18:24:21,723 DEBUG [StoreOpener-d1ad74f79ba4d7fd1619c83a55106331-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,723 INFO [StoreOpener-140f5077d3b730b643b558ebe8a13abd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 140f5077d3b730b643b558ebe8a13abd columnFamilyName colfamily21 2024-11-18T18:24:21,724 DEBUG [StoreOpener-140f5077d3b730b643b558ebe8a13abd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,724 INFO [StoreOpener-d1ad74f79ba4d7fd1619c83a55106331-1 {}] regionserver.HStore(327): Store=d1ad74f79ba4d7fd1619c83a55106331/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,724 INFO [StoreOpener-d1ad74f79ba4d7fd1619c83a55106331-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region d1ad74f79ba4d7fd1619c83a55106331 2024-11-18T18:24:21,725 INFO [StoreOpener-140f5077d3b730b643b558ebe8a13abd-1 {}] regionserver.HStore(327): Store=140f5077d3b730b643b558ebe8a13abd/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,725 INFO [StoreOpener-140f5077d3b730b643b558ebe8a13abd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 140f5077d3b730b643b558ebe8a13abd 2024-11-18T18:24:21,725 INFO [StoreOpener-579ee42a94597fdc1db9a4820a53ee64-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 579ee42a94597fdc1db9a4820a53ee64 2024-11-18T18:24:21,727 INFO [StoreOpener-d1ad74f79ba4d7fd1619c83a55106331-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1ad74f79ba4d7fd1619c83a55106331 columnFamilyName colfamily21 2024-11-18T18:24:21,727 DEBUG [StoreOpener-d1ad74f79ba4d7fd1619c83a55106331-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,727 INFO [StoreOpener-d1ad74f79ba4d7fd1619c83a55106331-1 {}] regionserver.HStore(327): Store=d1ad74f79ba4d7fd1619c83a55106331/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,728 INFO [StoreOpener-d1ad74f79ba4d7fd1619c83a55106331-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region d1ad74f79ba4d7fd1619c83a55106331 2024-11-18T18:24:21,728 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=c8b4dd2a6d8dd59085c547b55aac0de6, regionState=OPEN, openSeqNum=2, regionLocation=961a81da9827,42895,1731954257867 2024-11-18T18:24:21,728 INFO [StoreOpener-579ee42a94597fdc1db9a4820a53ee64-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 579ee42a94597fdc1db9a4820a53ee64 columnFamilyName colfamily11 2024-11-18T18:24:21,728 DEBUG [StoreOpener-579ee42a94597fdc1db9a4820a53ee64-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,729 INFO [StoreOpener-579ee42a94597fdc1db9a4820a53ee64-1 {}] regionserver.HStore(327): Store=579ee42a94597fdc1db9a4820a53ee64/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,731 INFO [StoreOpener-579ee42a94597fdc1db9a4820a53ee64-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 579ee42a94597fdc1db9a4820a53ee64 2024-11-18T18:24:21,731 INFO [StoreOpener-140f5077d3b730b643b558ebe8a13abd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 140f5077d3b730b643b558ebe8a13abd columnFamilyName colfamily31 2024-11-18T18:24:21,731 DEBUG [StoreOpener-140f5077d3b730b643b558ebe8a13abd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,732 INFO [StoreOpener-d1ad74f79ba4d7fd1619c83a55106331-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1ad74f79ba4d7fd1619c83a55106331 columnFamilyName colfamily31 2024-11-18T18:24:21,732 DEBUG [StoreOpener-d1ad74f79ba4d7fd1619c83a55106331-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,732 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure c8b4dd2a6d8dd59085c547b55aac0de6, server=961a81da9827,42895,1731954257867 because future has completed 2024-11-18T18:24:21,732 INFO [StoreOpener-140f5077d3b730b643b558ebe8a13abd-1 {}] regionserver.HStore(327): Store=140f5077d3b730b643b558ebe8a13abd/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,732 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for 140f5077d3b730b643b558ebe8a13abd 2024-11-18T18:24:21,733 INFO [StoreOpener-d1ad74f79ba4d7fd1619c83a55106331-1 {}] regionserver.HStore(327): Store=d1ad74f79ba4d7fd1619c83a55106331/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,733 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for d1ad74f79ba4d7fd1619c83a55106331 2024-11-18T18:24:21,734 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/140f5077d3b730b643b558ebe8a13abd 2024-11-18T18:24:21,734 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d1ad74f79ba4d7fd1619c83a55106331 2024-11-18T18:24:21,734 INFO [StoreOpener-579ee42a94597fdc1db9a4820a53ee64-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 579ee42a94597fdc1db9a4820a53ee64 columnFamilyName colfamily21 2024-11-18T18:24:21,734 DEBUG [StoreOpener-579ee42a94597fdc1db9a4820a53ee64-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,735 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d1ad74f79ba4d7fd1619c83a55106331 2024-11-18T18:24:21,735 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/140f5077d3b730b643b558ebe8a13abd 2024-11-18T18:24:21,736 INFO [StoreOpener-579ee42a94597fdc1db9a4820a53ee64-1 {}] regionserver.HStore(327): Store=579ee42a94597fdc1db9a4820a53ee64/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,737 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for d1ad74f79ba4d7fd1619c83a55106331 2024-11-18T18:24:21,737 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for 140f5077d3b730b643b558ebe8a13abd 2024-11-18T18:24:21,737 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for d1ad74f79ba4d7fd1619c83a55106331 2024-11-18T18:24:21,737 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for 140f5077d3b730b643b558ebe8a13abd 2024-11-18T18:24:21,738 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T18:24:21,738 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T18:24:21,738 INFO [StoreOpener-579ee42a94597fdc1db9a4820a53ee64-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 579ee42a94597fdc1db9a4820a53ee64 2024-11-18T18:24:21,745 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for 140f5077d3b730b643b558ebe8a13abd 2024-11-18T18:24:21,746 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for d1ad74f79ba4d7fd1619c83a55106331 2024-11-18T18:24:21,746 INFO [StoreOpener-579ee42a94597fdc1db9a4820a53ee64-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 579ee42a94597fdc1db9a4820a53ee64 columnFamilyName colfamily31 2024-11-18T18:24:21,747 DEBUG [StoreOpener-579ee42a94597fdc1db9a4820a53ee64-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,747 INFO [StoreOpener-579ee42a94597fdc1db9a4820a53ee64-1 {}] regionserver.HStore(327): Store=579ee42a94597fdc1db9a4820a53ee64/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,747 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for 579ee42a94597fdc1db9a4820a53ee64 2024-11-18T18:24:21,749 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/579ee42a94597fdc1db9a4820a53ee64 2024-11-18T18:24:21,749 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/579ee42a94597fdc1db9a4820a53ee64 2024-11-18T18:24:21,751 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for 579ee42a94597fdc1db9a4820a53ee64 2024-11-18T18:24:21,751 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for 579ee42a94597fdc1db9a4820a53ee64 2024-11-18T18:24:21,752 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T18:24:21,753 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for 579ee42a94597fdc1db9a4820a53ee64 2024-11-18T18:24:21,767 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/140f5077d3b730b643b558ebe8a13abd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T18:24:21,767 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=11 2024-11-18T18:24:21,767 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure c8b4dd2a6d8dd59085c547b55aac0de6, server=961a81da9827,42895,1731954257867 in 486 msec 2024-11-18T18:24:21,767 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d1ad74f79ba4d7fd1619c83a55106331/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T18:24:21,768 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened 140f5077d3b730b643b558ebe8a13abd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65229082, jitterRate=-0.028010934591293335}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T18:24:21,768 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 140f5077d3b730b643b558ebe8a13abd 2024-11-18T18:24:21,768 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for 140f5077d3b730b643b558ebe8a13abd: Running coprocessor pre-open hook at 1731954261703Writing region info on filesystem at 1731954261703Initializing all the Stores at 1731954261706 (+3 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261706Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261717 (+11 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261717Cleaning up temporary data from old regions at 1731954261738 (+21 ms)Running coprocessor post-open hooks at 1731954261768 (+30 ms)Region opened successfully at 1731954261768 2024-11-18T18:24:21,769 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened d1ad74f79ba4d7fd1619c83a55106331; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63994363, jitterRate=-0.04640968143939972}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T18:24:21,769 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d1ad74f79ba4d7fd1619c83a55106331 2024-11-18T18:24:21,769 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for d1ad74f79ba4d7fd1619c83a55106331: Running coprocessor pre-open hook at 1731954261686Writing region info on filesystem at 1731954261686Initializing all the Stores at 1731954261688 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261688Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261714 (+26 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261714Cleaning up temporary data from old regions at 1731954261737 (+23 ms)Running coprocessor post-open hooks at 1731954261769 (+32 ms)Region opened successfully at 1731954261769 2024-11-18T18:24:21,769 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731954260238.140f5077d3b730b643b558ebe8a13abd., pid=42, masterSystemTime=1731954261458 2024-11-18T18:24:21,770 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731954260238.d1ad74f79ba4d7fd1619c83a55106331., pid=53, masterSystemTime=1731954261502 2024-11-18T18:24:21,771 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c8b4dd2a6d8dd59085c547b55aac0de6, ASSIGN in 701 msec 2024-11-18T18:24:21,772 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731954260238.140f5077d3b730b643b558ebe8a13abd. 2024-11-18T18:24:21,773 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731954260238.140f5077d3b730b643b558ebe8a13abd. 2024-11-18T18:24:21,773 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731954260238.8bb7ae0747eda2290799ca8fa83f9ede. 2024-11-18T18:24:21,773 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => 8bb7ae0747eda2290799ca8fa83f9ede, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731954260238.8bb7ae0747eda2290799ca8fa83f9ede.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-18T18:24:21,774 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 8bb7ae0747eda2290799ca8fa83f9ede 2024-11-18T18:24:21,774 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731954260238.8bb7ae0747eda2290799ca8fa83f9ede.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:21,774 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=140f5077d3b730b643b558ebe8a13abd, regionState=OPEN, openSeqNum=2, regionLocation=961a81da9827,38709,1731954257723 2024-11-18T18:24:21,774 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for 8bb7ae0747eda2290799ca8fa83f9ede 2024-11-18T18:24:21,774 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for 8bb7ae0747eda2290799ca8fa83f9ede 2024-11-18T18:24:21,775 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731954260238.d1ad74f79ba4d7fd1619c83a55106331. 2024-11-18T18:24:21,777 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731954260238.d1ad74f79ba4d7fd1619c83a55106331. 2024-11-18T18:24:21,778 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731954260238.5e69ef2147d87d1b83dbcbb4705cbabe. 2024-11-18T18:24:21,778 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => 5e69ef2147d87d1b83dbcbb4705cbabe, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731954260238.5e69ef2147d87d1b83dbcbb4705cbabe.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-18T18:24:21,778 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 140f5077d3b730b643b558ebe8a13abd, server=961a81da9827,38709,1731954257723 because future has completed 2024-11-18T18:24:21,778 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 5e69ef2147d87d1b83dbcbb4705cbabe 2024-11-18T18:24:21,778 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731954260238.5e69ef2147d87d1b83dbcbb4705cbabe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:21,779 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for 5e69ef2147d87d1b83dbcbb4705cbabe 2024-11-18T18:24:21,779 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for 5e69ef2147d87d1b83dbcbb4705cbabe 2024-11-18T18:24:21,780 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/579ee42a94597fdc1db9a4820a53ee64/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T18:24:21,780 INFO [StoreOpener-8bb7ae0747eda2290799ca8fa83f9ede-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 8bb7ae0747eda2290799ca8fa83f9ede 2024-11-18T18:24:21,781 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened 579ee42a94597fdc1db9a4820a53ee64; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68475632, jitterRate=0.020366430282592773}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T18:24:21,781 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 579ee42a94597fdc1db9a4820a53ee64 2024-11-18T18:24:21,781 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for 579ee42a94597fdc1db9a4820a53ee64: Running coprocessor pre-open hook at 1731954261715Writing region info on filesystem at 1731954261715Initializing all the Stores at 1731954261716 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261716Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261724 (+8 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261724Cleaning up temporary data from old regions at 1731954261751 (+27 ms)Running coprocessor post-open hooks at 1731954261781 (+30 ms)Region opened successfully at 1731954261781 2024-11-18T18:24:21,782 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731954260238.579ee42a94597fdc1db9a4820a53ee64., pid=31, masterSystemTime=1731954261396 2024-11-18T18:24:21,783 INFO [StoreOpener-8bb7ae0747eda2290799ca8fa83f9ede-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8bb7ae0747eda2290799ca8fa83f9ede columnFamilyName colfamily11 2024-11-18T18:24:21,784 DEBUG [StoreOpener-8bb7ae0747eda2290799ca8fa83f9ede-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,784 INFO [StoreOpener-8bb7ae0747eda2290799ca8fa83f9ede-1 {}] regionserver.HStore(327): Store=8bb7ae0747eda2290799ca8fa83f9ede/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,785 INFO [StoreOpener-8bb7ae0747eda2290799ca8fa83f9ede-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 8bb7ae0747eda2290799ca8fa83f9ede 2024-11-18T18:24:21,786 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731954260238.579ee42a94597fdc1db9a4820a53ee64. 2024-11-18T18:24:21,786 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731954260238.579ee42a94597fdc1db9a4820a53ee64. 2024-11-18T18:24:21,786 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731954260238.fb8fa4293ea5e58c18dbc5b978355ca9. 2024-11-18T18:24:21,786 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => fb8fa4293ea5e58c18dbc5b978355ca9, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731954260238.fb8fa4293ea5e58c18dbc5b978355ca9.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-18T18:24:21,786 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true fb8fa4293ea5e58c18dbc5b978355ca9 2024-11-18T18:24:21,787 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731954260238.fb8fa4293ea5e58c18dbc5b978355ca9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:21,787 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for fb8fa4293ea5e58c18dbc5b978355ca9 2024-11-18T18:24:21,787 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for fb8fa4293ea5e58c18dbc5b978355ca9 2024-11-18T18:24:21,789 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=19 2024-11-18T18:24:21,789 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure 140f5077d3b730b643b558ebe8a13abd, server=961a81da9827,38709,1731954257723 in 466 msec 2024-11-18T18:24:21,789 INFO [StoreOpener-8bb7ae0747eda2290799ca8fa83f9ede-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8bb7ae0747eda2290799ca8fa83f9ede columnFamilyName colfamily21 2024-11-18T18:24:21,789 DEBUG [StoreOpener-8bb7ae0747eda2290799ca8fa83f9ede-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,790 INFO [StoreOpener-8bb7ae0747eda2290799ca8fa83f9ede-1 {}] regionserver.HStore(327): Store=8bb7ae0747eda2290799ca8fa83f9ede/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,790 INFO [StoreOpener-8bb7ae0747eda2290799ca8fa83f9ede-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 8bb7ae0747eda2290799ca8fa83f9ede 2024-11-18T18:24:21,792 INFO [StoreOpener-8bb7ae0747eda2290799ca8fa83f9ede-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8bb7ae0747eda2290799ca8fa83f9ede columnFamilyName colfamily31 2024-11-18T18:24:21,793 DEBUG [StoreOpener-8bb7ae0747eda2290799ca8fa83f9ede-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,793 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=140f5077d3b730b643b558ebe8a13abd, ASSIGN in 723 msec 2024-11-18T18:24:21,793 INFO [StoreOpener-8bb7ae0747eda2290799ca8fa83f9ede-1 {}] regionserver.HStore(327): Store=8bb7ae0747eda2290799ca8fa83f9ede/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,793 INFO [StoreOpener-5e69ef2147d87d1b83dbcbb4705cbabe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 5e69ef2147d87d1b83dbcbb4705cbabe 2024-11-18T18:24:21,794 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for 8bb7ae0747eda2290799ca8fa83f9ede 2024-11-18T18:24:21,795 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=579ee42a94597fdc1db9a4820a53ee64, regionState=OPEN, openSeqNum=2, regionLocation=961a81da9827,42895,1731954257867 2024-11-18T18:24:21,795 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8bb7ae0747eda2290799ca8fa83f9ede 2024-11-18T18:24:21,796 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8bb7ae0747eda2290799ca8fa83f9ede 2024-11-18T18:24:21,797 INFO [StoreOpener-5e69ef2147d87d1b83dbcbb4705cbabe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5e69ef2147d87d1b83dbcbb4705cbabe columnFamilyName colfamily11 2024-11-18T18:24:21,797 DEBUG [StoreOpener-5e69ef2147d87d1b83dbcbb4705cbabe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,798 INFO [StoreOpener-5e69ef2147d87d1b83dbcbb4705cbabe-1 {}] regionserver.HStore(327): Store=5e69ef2147d87d1b83dbcbb4705cbabe/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,798 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for 8bb7ae0747eda2290799ca8fa83f9ede 2024-11-18T18:24:21,799 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for 8bb7ae0747eda2290799ca8fa83f9ede 2024-11-18T18:24:21,799 INFO [StoreOpener-5e69ef2147d87d1b83dbcbb4705cbabe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 5e69ef2147d87d1b83dbcbb4705cbabe 2024-11-18T18:24:21,799 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 579ee42a94597fdc1db9a4820a53ee64, server=961a81da9827,42895,1731954257867 because future has completed 2024-11-18T18:24:21,799 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T18:24:21,801 INFO [StoreOpener-fb8fa4293ea5e58c18dbc5b978355ca9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region fb8fa4293ea5e58c18dbc5b978355ca9 2024-11-18T18:24:21,802 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for 8bb7ae0747eda2290799ca8fa83f9ede 2024-11-18T18:24:21,805 INFO [StoreOpener-fb8fa4293ea5e58c18dbc5b978355ca9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fb8fa4293ea5e58c18dbc5b978355ca9 columnFamilyName colfamily11 2024-11-18T18:24:21,805 DEBUG [StoreOpener-fb8fa4293ea5e58c18dbc5b978355ca9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,805 INFO [StoreOpener-5e69ef2147d87d1b83dbcbb4705cbabe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5e69ef2147d87d1b83dbcbb4705cbabe columnFamilyName colfamily21 2024-11-18T18:24:21,805 DEBUG [StoreOpener-5e69ef2147d87d1b83dbcbb4705cbabe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,806 INFO [StoreOpener-fb8fa4293ea5e58c18dbc5b978355ca9-1 {}] regionserver.HStore(327): Store=fb8fa4293ea5e58c18dbc5b978355ca9/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,806 INFO [StoreOpener-fb8fa4293ea5e58c18dbc5b978355ca9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region fb8fa4293ea5e58c18dbc5b978355ca9 2024-11-18T18:24:21,806 INFO [StoreOpener-5e69ef2147d87d1b83dbcbb4705cbabe-1 {}] regionserver.HStore(327): Store=5e69ef2147d87d1b83dbcbb4705cbabe/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,806 INFO [StoreOpener-5e69ef2147d87d1b83dbcbb4705cbabe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 5e69ef2147d87d1b83dbcbb4705cbabe 2024-11-18T18:24:21,807 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=28 2024-11-18T18:24:21,807 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure 579ee42a94597fdc1db9a4820a53ee64, server=961a81da9827,42895,1731954257867 in 560 msec 2024-11-18T18:24:21,808 INFO [StoreOpener-fb8fa4293ea5e58c18dbc5b978355ca9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fb8fa4293ea5e58c18dbc5b978355ca9 columnFamilyName colfamily21 2024-11-18T18:24:21,808 DEBUG [StoreOpener-fb8fa4293ea5e58c18dbc5b978355ca9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,808 INFO [StoreOpener-5e69ef2147d87d1b83dbcbb4705cbabe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5e69ef2147d87d1b83dbcbb4705cbabe columnFamilyName colfamily31 2024-11-18T18:24:21,808 DEBUG [StoreOpener-5e69ef2147d87d1b83dbcbb4705cbabe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,809 INFO [StoreOpener-5e69ef2147d87d1b83dbcbb4705cbabe-1 {}] regionserver.HStore(327): Store=5e69ef2147d87d1b83dbcbb4705cbabe/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,809 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for 5e69ef2147d87d1b83dbcbb4705cbabe 2024-11-18T18:24:21,810 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5e69ef2147d87d1b83dbcbb4705cbabe 2024-11-18T18:24:21,811 INFO [StoreOpener-fb8fa4293ea5e58c18dbc5b978355ca9-1 {}] regionserver.HStore(327): Store=fb8fa4293ea5e58c18dbc5b978355ca9/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,811 INFO [StoreOpener-fb8fa4293ea5e58c18dbc5b978355ca9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region fb8fa4293ea5e58c18dbc5b978355ca9 2024-11-18T18:24:21,811 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5e69ef2147d87d1b83dbcbb4705cbabe 2024-11-18T18:24:21,811 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=579ee42a94597fdc1db9a4820a53ee64, ASSIGN in 741 msec 2024-11-18T18:24:21,813 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for 5e69ef2147d87d1b83dbcbb4705cbabe 2024-11-18T18:24:21,813 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for 5e69ef2147d87d1b83dbcbb4705cbabe 2024-11-18T18:24:21,814 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T18:24:21,814 INFO [StoreOpener-fb8fa4293ea5e58c18dbc5b978355ca9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fb8fa4293ea5e58c18dbc5b978355ca9 columnFamilyName colfamily31 2024-11-18T18:24:21,814 DEBUG [StoreOpener-fb8fa4293ea5e58c18dbc5b978355ca9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,815 INFO [StoreOpener-fb8fa4293ea5e58c18dbc5b978355ca9-1 {}] regionserver.HStore(327): Store=fb8fa4293ea5e58c18dbc5b978355ca9/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,815 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for fb8fa4293ea5e58c18dbc5b978355ca9 2024-11-18T18:24:21,815 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=d1ad74f79ba4d7fd1619c83a55106331, regionState=OPEN, openSeqNum=2, regionLocation=961a81da9827,36379,1731954257826 2024-11-18T18:24:21,816 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fb8fa4293ea5e58c18dbc5b978355ca9 2024-11-18T18:24:21,817 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fb8fa4293ea5e58c18dbc5b978355ca9 2024-11-18T18:24:21,819 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for fb8fa4293ea5e58c18dbc5b978355ca9 2024-11-18T18:24:21,819 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for fb8fa4293ea5e58c18dbc5b978355ca9 2024-11-18T18:24:21,819 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for 5e69ef2147d87d1b83dbcbb4705cbabe 2024-11-18T18:24:21,820 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=53, ppid=25, state=RUNNABLE, hasLock=true; OpenRegionProcedure d1ad74f79ba4d7fd1619c83a55106331, server=961a81da9827,36379,1731954257826 2024-11-18T18:24:21,820 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T18:24:21,822 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8bb7ae0747eda2290799ca8fa83f9ede/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T18:24:21,823 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened 8bb7ae0747eda2290799ca8fa83f9ede; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67378389, jitterRate=0.0040162354707717896}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T18:24:21,823 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8bb7ae0747eda2290799ca8fa83f9ede 2024-11-18T18:24:21,823 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for 8bb7ae0747eda2290799ca8fa83f9ede: Running coprocessor pre-open hook at 1731954261774Writing region info on filesystem at 1731954261774Initializing all the Stores at 1731954261777 (+3 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261777Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261779 (+2 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261779Cleaning up temporary data from old regions at 1731954261799 (+20 ms)Running coprocessor post-open hooks at 1731954261823 (+24 ms)Region opened successfully at 1731954261823 2024-11-18T18:24:21,823 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for fb8fa4293ea5e58c18dbc5b978355ca9 2024-11-18T18:24:21,824 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731954260238.8bb7ae0747eda2290799ca8fa83f9ede., pid=55, masterSystemTime=1731954261458 2024-11-18T18:24:21,827 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=25 2024-11-18T18:24:21,827 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure d1ad74f79ba4d7fd1619c83a55106331, server=961a81da9827,36379,1731954257826 in 436 msec 2024-11-18T18:24:21,828 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731954260238.8bb7ae0747eda2290799ca8fa83f9ede. 2024-11-18T18:24:21,829 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731954260238.8bb7ae0747eda2290799ca8fa83f9ede. 2024-11-18T18:24:21,829 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731954260238.97377a91d35f8b242120f603c456c41e. 2024-11-18T18:24:21,829 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 97377a91d35f8b242120f603c456c41e, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731954260238.97377a91d35f8b242120f603c456c41e.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-18T18:24:21,830 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 97377a91d35f8b242120f603c456c41e 2024-11-18T18:24:21,830 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d1ad74f79ba4d7fd1619c83a55106331, ASSIGN in 761 msec 2024-11-18T18:24:21,830 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731954260238.97377a91d35f8b242120f603c456c41e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:21,830 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 97377a91d35f8b242120f603c456c41e 2024-11-18T18:24:21,830 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 97377a91d35f8b242120f603c456c41e 2024-11-18T18:24:21,833 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8bb7ae0747eda2290799ca8fa83f9ede, regionState=OPEN, openSeqNum=2, regionLocation=961a81da9827,38709,1731954257723 2024-11-18T18:24:21,833 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fb8fa4293ea5e58c18dbc5b978355ca9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T18:24:21,834 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened fb8fa4293ea5e58c18dbc5b978355ca9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61824085, jitterRate=-0.07874934375286102}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T18:24:21,834 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fb8fa4293ea5e58c18dbc5b978355ca9 2024-11-18T18:24:21,834 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for fb8fa4293ea5e58c18dbc5b978355ca9: Running coprocessor pre-open hook at 1731954261787Writing region info on filesystem at 1731954261787Initializing all the Stores at 1731954261788 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261788Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261795 (+7 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261795Cleaning up temporary data from old regions at 1731954261819 (+24 ms)Running coprocessor post-open hooks at 1731954261834 (+15 ms)Region opened successfully at 1731954261834 2024-11-18T18:24:21,834 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5e69ef2147d87d1b83dbcbb4705cbabe/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T18:24:21,835 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731954260238.fb8fa4293ea5e58c18dbc5b978355ca9., pid=32, masterSystemTime=1731954261396 2024-11-18T18:24:21,836 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened 5e69ef2147d87d1b83dbcbb4705cbabe; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65456970, jitterRate=-0.02461513876914978}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T18:24:21,836 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5e69ef2147d87d1b83dbcbb4705cbabe 2024-11-18T18:24:21,836 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for 5e69ef2147d87d1b83dbcbb4705cbabe: Running coprocessor pre-open hook at 1731954261779Writing region info on filesystem at 1731954261779Initializing all the Stores at 1731954261781 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261781Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261793 (+12 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261793Cleaning up temporary data from old regions at 1731954261813 (+20 ms)Running coprocessor post-open hooks at 1731954261836 (+23 ms)Region opened successfully at 1731954261836 2024-11-18T18:24:21,838 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731954260238.5e69ef2147d87d1b83dbcbb4705cbabe., pid=52, masterSystemTime=1731954261502 2024-11-18T18:24:21,839 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8bb7ae0747eda2290799ca8fa83f9ede, server=961a81da9827,38709,1731954257723 because future has completed 2024-11-18T18:24:21,839 INFO [StoreOpener-97377a91d35f8b242120f603c456c41e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 97377a91d35f8b242120f603c456c41e 2024-11-18T18:24:21,841 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731954260238.fb8fa4293ea5e58c18dbc5b978355ca9. 2024-11-18T18:24:21,841 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731954260238.fb8fa4293ea5e58c18dbc5b978355ca9. 2024-11-18T18:24:21,841 INFO [StoreOpener-97377a91d35f8b242120f603c456c41e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97377a91d35f8b242120f603c456c41e columnFamilyName colfamily11 2024-11-18T18:24:21,841 DEBUG [StoreOpener-97377a91d35f8b242120f603c456c41e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,842 INFO [StoreOpener-97377a91d35f8b242120f603c456c41e-1 {}] regionserver.HStore(327): Store=97377a91d35f8b242120f603c456c41e/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,842 INFO [StoreOpener-97377a91d35f8b242120f603c456c41e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 97377a91d35f8b242120f603c456c41e 2024-11-18T18:24:21,842 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731954260238.5e69ef2147d87d1b83dbcbb4705cbabe. 2024-11-18T18:24:21,843 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731954260238.5e69ef2147d87d1b83dbcbb4705cbabe. 2024-11-18T18:24:21,843 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731954260238.bcf8695060a326b7d24615ba6cb932b1. 2024-11-18T18:24:21,843 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => bcf8695060a326b7d24615ba6cb932b1, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731954260238.bcf8695060a326b7d24615ba6cb932b1.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-18T18:24:21,844 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true bcf8695060a326b7d24615ba6cb932b1 2024-11-18T18:24:21,844 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731954260238.bcf8695060a326b7d24615ba6cb932b1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:21,844 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for bcf8695060a326b7d24615ba6cb932b1 2024-11-18T18:24:21,844 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for bcf8695060a326b7d24615ba6cb932b1 2024-11-18T18:24:21,845 INFO [StoreOpener-97377a91d35f8b242120f603c456c41e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97377a91d35f8b242120f603c456c41e columnFamilyName colfamily21 2024-11-18T18:24:21,845 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=5 2024-11-18T18:24:21,845 DEBUG [StoreOpener-97377a91d35f8b242120f603c456c41e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,846 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 8bb7ae0747eda2290799ca8fa83f9ede, server=961a81da9827,38709,1731954257723 in 448 msec 2024-11-18T18:24:21,846 INFO [StoreOpener-97377a91d35f8b242120f603c456c41e-1 {}] regionserver.HStore(327): Store=97377a91d35f8b242120f603c456c41e/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,847 INFO [StoreOpener-97377a91d35f8b242120f603c456c41e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 97377a91d35f8b242120f603c456c41e 2024-11-18T18:24:21,848 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8bb7ae0747eda2290799ca8fa83f9ede, ASSIGN in 780 msec 2024-11-18T18:24:21,849 INFO [StoreOpener-97377a91d35f8b242120f603c456c41e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97377a91d35f8b242120f603c456c41e columnFamilyName colfamily31 2024-11-18T18:24:21,849 DEBUG [StoreOpener-97377a91d35f8b242120f603c456c41e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,849 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=5e69ef2147d87d1b83dbcbb4705cbabe, regionState=OPEN, openSeqNum=2, regionLocation=961a81da9827,36379,1731954257826 2024-11-18T18:24:21,850 INFO [StoreOpener-97377a91d35f8b242120f603c456c41e-1 {}] regionserver.HStore(327): Store=97377a91d35f8b242120f603c456c41e/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,850 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 97377a91d35f8b242120f603c456c41e 2024-11-18T18:24:21,852 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/97377a91d35f8b242120f603c456c41e 2024-11-18T18:24:21,853 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/97377a91d35f8b242120f603c456c41e 2024-11-18T18:24:21,854 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5e69ef2147d87d1b83dbcbb4705cbabe, server=961a81da9827,36379,1731954257826 because future has completed 2024-11-18T18:24:21,855 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 97377a91d35f8b242120f603c456c41e 2024-11-18T18:24:21,855 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 97377a91d35f8b242120f603c456c41e 2024-11-18T18:24:21,857 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T18:24:21,857 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=fb8fa4293ea5e58c18dbc5b978355ca9, regionState=OPEN, openSeqNum=2, regionLocation=961a81da9827,42895,1731954257867 2024-11-18T18:24:21,858 INFO [StoreOpener-bcf8695060a326b7d24615ba6cb932b1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region bcf8695060a326b7d24615ba6cb932b1 2024-11-18T18:24:21,861 INFO [StoreOpener-bcf8695060a326b7d24615ba6cb932b1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bcf8695060a326b7d24615ba6cb932b1 columnFamilyName colfamily11 2024-11-18T18:24:21,861 DEBUG [StoreOpener-bcf8695060a326b7d24615ba6cb932b1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,861 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 97377a91d35f8b242120f603c456c41e 2024-11-18T18:24:21,862 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure fb8fa4293ea5e58c18dbc5b978355ca9, server=961a81da9827,42895,1731954257867 because future has completed 2024-11-18T18:24:21,862 INFO [StoreOpener-bcf8695060a326b7d24615ba6cb932b1-1 {}] regionserver.HStore(327): Store=bcf8695060a326b7d24615ba6cb932b1/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,863 INFO [StoreOpener-bcf8695060a326b7d24615ba6cb932b1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region bcf8695060a326b7d24615ba6cb932b1 2024-11-18T18:24:21,870 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=21 2024-11-18T18:24:21,870 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure fb8fa4293ea5e58c18dbc5b978355ca9, server=961a81da9827,42895,1731954257867 in 611 msec 2024-11-18T18:24:21,873 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fb8fa4293ea5e58c18dbc5b978355ca9, ASSIGN in 805 msec 2024-11-18T18:24:21,881 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=20 2024-11-18T18:24:21,881 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 5e69ef2147d87d1b83dbcbb4705cbabe, server=961a81da9827,36379,1731954257826 in 496 msec 2024-11-18T18:24:21,884 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5e69ef2147d87d1b83dbcbb4705cbabe, ASSIGN in 815 msec 2024-11-18T18:24:21,885 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/97377a91d35f8b242120f603c456c41e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T18:24:21,886 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 97377a91d35f8b242120f603c456c41e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66960505, jitterRate=-0.0022107213735580444}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T18:24:21,886 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 97377a91d35f8b242120f603c456c41e 2024-11-18T18:24:21,886 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 97377a91d35f8b242120f603c456c41e: Running coprocessor pre-open hook at 1731954261830Writing region info on filesystem at 1731954261830Initializing all the Stores at 1731954261831 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261831Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261833 (+2 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261833Cleaning up temporary data from old regions at 1731954261855 (+22 ms)Running coprocessor post-open hooks at 1731954261886 (+31 ms)Region opened successfully at 1731954261886 2024-11-18T18:24:21,887 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731954260238.97377a91d35f8b242120f603c456c41e., pid=41, masterSystemTime=1731954261458 2024-11-18T18:24:21,889 INFO [StoreOpener-bcf8695060a326b7d24615ba6cb932b1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bcf8695060a326b7d24615ba6cb932b1 columnFamilyName colfamily21 2024-11-18T18:24:21,889 DEBUG [StoreOpener-bcf8695060a326b7d24615ba6cb932b1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,893 INFO [StoreOpener-bcf8695060a326b7d24615ba6cb932b1-1 {}] regionserver.HStore(327): Store=bcf8695060a326b7d24615ba6cb932b1/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,893 INFO [StoreOpener-bcf8695060a326b7d24615ba6cb932b1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region bcf8695060a326b7d24615ba6cb932b1 2024-11-18T18:24:21,896 INFO [StoreOpener-bcf8695060a326b7d24615ba6cb932b1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bcf8695060a326b7d24615ba6cb932b1 columnFamilyName colfamily31 2024-11-18T18:24:21,897 DEBUG [StoreOpener-bcf8695060a326b7d24615ba6cb932b1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,897 INFO [StoreOpener-bcf8695060a326b7d24615ba6cb932b1-1 {}] regionserver.HStore(327): Store=bcf8695060a326b7d24615ba6cb932b1/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,898 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for bcf8695060a326b7d24615ba6cb932b1 2024-11-18T18:24:21,899 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bcf8695060a326b7d24615ba6cb932b1 2024-11-18T18:24:21,901 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bcf8695060a326b7d24615ba6cb932b1 2024-11-18T18:24:21,902 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731954260238.97377a91d35f8b242120f603c456c41e. 2024-11-18T18:24:21,902 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731954260238.97377a91d35f8b242120f603c456c41e. 2024-11-18T18:24:21,903 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for bcf8695060a326b7d24615ba6cb932b1 2024-11-18T18:24:21,903 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=97377a91d35f8b242120f603c456c41e, regionState=OPEN, openSeqNum=2, regionLocation=961a81da9827,38709,1731954257723 2024-11-18T18:24:21,903 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for bcf8695060a326b7d24615ba6cb932b1 2024-11-18T18:24:21,904 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T18:24:21,907 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for bcf8695060a326b7d24615ba6cb932b1 2024-11-18T18:24:21,908 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 97377a91d35f8b242120f603c456c41e, server=961a81da9827,38709,1731954257723 because future has completed 2024-11-18T18:24:21,924 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=16 2024-11-18T18:24:21,924 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 97377a91d35f8b242120f603c456c41e, server=961a81da9827,38709,1731954257723 in 607 msec 2024-11-18T18:24:21,926 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bcf8695060a326b7d24615ba6cb932b1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T18:24:21,928 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=97377a91d35f8b242120f603c456c41e, ASSIGN in 858 msec 2024-11-18T18:24:21,938 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened bcf8695060a326b7d24615ba6cb932b1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60124651, jitterRate=-0.10407288372516632}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T18:24:21,938 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bcf8695060a326b7d24615ba6cb932b1 2024-11-18T18:24:21,938 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for bcf8695060a326b7d24615ba6cb932b1: Running coprocessor pre-open hook at 1731954261844Writing region info on filesystem at 1731954261844Initializing all the Stores at 1731954261846 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261846Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261858 (+12 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261858Cleaning up temporary data from old regions at 1731954261903 (+45 ms)Running coprocessor post-open hooks at 1731954261938 (+35 ms)Region opened successfully at 1731954261938 2024-11-18T18:24:21,942 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731954260238.bcf8695060a326b7d24615ba6cb932b1., pid=49, masterSystemTime=1731954261502 2024-11-18T18:24:21,947 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731954260238.bcf8695060a326b7d24615ba6cb932b1. 2024-11-18T18:24:21,947 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731954260238.bcf8695060a326b7d24615ba6cb932b1. 2024-11-18T18:24:21,947 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731954260238.a244d0ab318eb4dd77bc5235828d9d51. 2024-11-18T18:24:21,948 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => a244d0ab318eb4dd77bc5235828d9d51, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731954260238.a244d0ab318eb4dd77bc5235828d9d51.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-18T18:24:21,948 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true a244d0ab318eb4dd77bc5235828d9d51 2024-11-18T18:24:21,948 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731954260238.a244d0ab318eb4dd77bc5235828d9d51.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:21,949 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for a244d0ab318eb4dd77bc5235828d9d51 2024-11-18T18:24:21,949 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for a244d0ab318eb4dd77bc5235828d9d51 2024-11-18T18:24:21,950 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=bcf8695060a326b7d24615ba6cb932b1, regionState=OPEN, openSeqNum=2, regionLocation=961a81da9827,36379,1731954257826 2024-11-18T18:24:21,953 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure bcf8695060a326b7d24615ba6cb932b1, server=961a81da9827,36379,1731954257826 because future has completed 2024-11-18T18:24:21,960 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=9 2024-11-18T18:24:21,961 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure bcf8695060a326b7d24615ba6cb932b1, server=961a81da9827,36379,1731954257826 in 590 msec 2024-11-18T18:24:21,964 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=bcf8695060a326b7d24615ba6cb932b1, ASSIGN in 895 msec 2024-11-18T18:24:21,986 INFO [StoreOpener-a244d0ab318eb4dd77bc5235828d9d51-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region a244d0ab318eb4dd77bc5235828d9d51 2024-11-18T18:24:21,990 INFO [StoreOpener-a244d0ab318eb4dd77bc5235828d9d51-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a244d0ab318eb4dd77bc5235828d9d51 columnFamilyName colfamily11 2024-11-18T18:24:21,990 DEBUG [StoreOpener-a244d0ab318eb4dd77bc5235828d9d51-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,991 INFO [StoreOpener-a244d0ab318eb4dd77bc5235828d9d51-1 {}] regionserver.HStore(327): Store=a244d0ab318eb4dd77bc5235828d9d51/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,991 INFO [StoreOpener-a244d0ab318eb4dd77bc5235828d9d51-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region a244d0ab318eb4dd77bc5235828d9d51 2024-11-18T18:24:21,993 INFO [StoreOpener-a244d0ab318eb4dd77bc5235828d9d51-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a244d0ab318eb4dd77bc5235828d9d51 columnFamilyName colfamily21 2024-11-18T18:24:21,993 DEBUG [StoreOpener-a244d0ab318eb4dd77bc5235828d9d51-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,994 INFO [StoreOpener-a244d0ab318eb4dd77bc5235828d9d51-1 {}] regionserver.HStore(327): Store=a244d0ab318eb4dd77bc5235828d9d51/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,994 INFO [StoreOpener-a244d0ab318eb4dd77bc5235828d9d51-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region a244d0ab318eb4dd77bc5235828d9d51 2024-11-18T18:24:21,997 INFO [StoreOpener-a244d0ab318eb4dd77bc5235828d9d51-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a244d0ab318eb4dd77bc5235828d9d51 columnFamilyName colfamily31 2024-11-18T18:24:21,997 DEBUG [StoreOpener-a244d0ab318eb4dd77bc5235828d9d51-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:21,998 INFO [StoreOpener-a244d0ab318eb4dd77bc5235828d9d51-1 {}] regionserver.HStore(327): Store=a244d0ab318eb4dd77bc5235828d9d51/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:21,998 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for a244d0ab318eb4dd77bc5235828d9d51 2024-11-18T18:24:21,999 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a244d0ab318eb4dd77bc5235828d9d51 2024-11-18T18:24:22,000 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a244d0ab318eb4dd77bc5235828d9d51 2024-11-18T18:24:22,005 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for a244d0ab318eb4dd77bc5235828d9d51 2024-11-18T18:24:22,005 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for a244d0ab318eb4dd77bc5235828d9d51 2024-11-18T18:24:22,007 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T18:24:22,009 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for a244d0ab318eb4dd77bc5235828d9d51 2024-11-18T18:24:22,018 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a244d0ab318eb4dd77bc5235828d9d51/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T18:24:22,019 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened a244d0ab318eb4dd77bc5235828d9d51; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62759840, jitterRate=-0.06480550765991211}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T18:24:22,019 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a244d0ab318eb4dd77bc5235828d9d51 2024-11-18T18:24:22,020 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for a244d0ab318eb4dd77bc5235828d9d51: Running coprocessor pre-open hook at 1731954261949Writing region info on filesystem at 1731954261949Initializing all the Stores at 1731954261953 (+4 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261953Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261986 (+33 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954261986Cleaning up temporary data from old regions at 1731954262005 (+19 ms)Running coprocessor post-open hooks at 1731954262019 (+14 ms)Region opened successfully at 1731954262019 2024-11-18T18:24:22,021 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731954260238.a244d0ab318eb4dd77bc5235828d9d51., pid=47, masterSystemTime=1731954261502 2024-11-18T18:24:22,027 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731954260238.a244d0ab318eb4dd77bc5235828d9d51. 2024-11-18T18:24:22,027 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731954260238.a244d0ab318eb4dd77bc5235828d9d51. 2024-11-18T18:24:22,028 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731954260238.c3e2f3fb6516fcaaef4d0a2c8b63842d. 2024-11-18T18:24:22,028 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => c3e2f3fb6516fcaaef4d0a2c8b63842d, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731954260238.c3e2f3fb6516fcaaef4d0a2c8b63842d.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-18T18:24:22,028 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=a244d0ab318eb4dd77bc5235828d9d51, regionState=OPEN, openSeqNum=2, regionLocation=961a81da9827,36379,1731954257826 2024-11-18T18:24:22,028 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true c3e2f3fb6516fcaaef4d0a2c8b63842d 2024-11-18T18:24:22,028 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731954260238.c3e2f3fb6516fcaaef4d0a2c8b63842d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:22,029 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for c3e2f3fb6516fcaaef4d0a2c8b63842d 2024-11-18T18:24:22,029 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for c3e2f3fb6516fcaaef4d0a2c8b63842d 2024-11-18T18:24:22,033 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure a244d0ab318eb4dd77bc5235828d9d51, server=961a81da9827,36379,1731954257826 because future has completed 2024-11-18T18:24:22,040 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=17 2024-11-18T18:24:22,040 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure a244d0ab318eb4dd77bc5235828d9d51, server=961a81da9827,36379,1731954257826 in 682 msec 2024-11-18T18:24:22,042 INFO [StoreOpener-c3e2f3fb6516fcaaef4d0a2c8b63842d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region c3e2f3fb6516fcaaef4d0a2c8b63842d 2024-11-18T18:24:22,043 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a244d0ab318eb4dd77bc5235828d9d51, ASSIGN in 974 msec 2024-11-18T18:24:22,045 INFO [StoreOpener-c3e2f3fb6516fcaaef4d0a2c8b63842d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c3e2f3fb6516fcaaef4d0a2c8b63842d columnFamilyName colfamily11 2024-11-18T18:24:22,046 DEBUG [StoreOpener-c3e2f3fb6516fcaaef4d0a2c8b63842d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:22,046 INFO [StoreOpener-c3e2f3fb6516fcaaef4d0a2c8b63842d-1 {}] regionserver.HStore(327): Store=c3e2f3fb6516fcaaef4d0a2c8b63842d/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:22,046 INFO [StoreOpener-c3e2f3fb6516fcaaef4d0a2c8b63842d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region c3e2f3fb6516fcaaef4d0a2c8b63842d 2024-11-18T18:24:22,048 INFO [StoreOpener-c3e2f3fb6516fcaaef4d0a2c8b63842d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c3e2f3fb6516fcaaef4d0a2c8b63842d columnFamilyName colfamily21 2024-11-18T18:24:22,048 DEBUG [StoreOpener-c3e2f3fb6516fcaaef4d0a2c8b63842d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:22,049 INFO [StoreOpener-c3e2f3fb6516fcaaef4d0a2c8b63842d-1 {}] regionserver.HStore(327): Store=c3e2f3fb6516fcaaef4d0a2c8b63842d/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:22,049 INFO [StoreOpener-c3e2f3fb6516fcaaef4d0a2c8b63842d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region c3e2f3fb6516fcaaef4d0a2c8b63842d 2024-11-18T18:24:22,051 INFO [StoreOpener-c3e2f3fb6516fcaaef4d0a2c8b63842d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c3e2f3fb6516fcaaef4d0a2c8b63842d columnFamilyName colfamily31 2024-11-18T18:24:22,051 DEBUG [StoreOpener-c3e2f3fb6516fcaaef4d0a2c8b63842d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:22,052 INFO [StoreOpener-c3e2f3fb6516fcaaef4d0a2c8b63842d-1 {}] regionserver.HStore(327): Store=c3e2f3fb6516fcaaef4d0a2c8b63842d/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:22,052 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for c3e2f3fb6516fcaaef4d0a2c8b63842d 2024-11-18T18:24:22,053 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c3e2f3fb6516fcaaef4d0a2c8b63842d 2024-11-18T18:24:22,054 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c3e2f3fb6516fcaaef4d0a2c8b63842d 2024-11-18T18:24:22,055 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for c3e2f3fb6516fcaaef4d0a2c8b63842d 2024-11-18T18:24:22,055 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for c3e2f3fb6516fcaaef4d0a2c8b63842d 2024-11-18T18:24:22,056 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T18:24:22,059 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for c3e2f3fb6516fcaaef4d0a2c8b63842d 2024-11-18T18:24:22,063 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c3e2f3fb6516fcaaef4d0a2c8b63842d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T18:24:22,064 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened c3e2f3fb6516fcaaef4d0a2c8b63842d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74860492, jitterRate=0.11550825834274292}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T18:24:22,064 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c3e2f3fb6516fcaaef4d0a2c8b63842d 2024-11-18T18:24:22,065 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for c3e2f3fb6516fcaaef4d0a2c8b63842d: Running coprocessor pre-open hook at 1731954262029Writing region info on filesystem at 1731954262029Initializing all the Stores at 1731954262031 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954262031Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954262041 (+10 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954262041Cleaning up temporary data from old regions at 1731954262055 (+14 ms)Running coprocessor post-open hooks at 1731954262064 (+9 ms)Region opened successfully at 1731954262065 (+1 ms) 2024-11-18T18:24:22,069 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731954260238.c3e2f3fb6516fcaaef4d0a2c8b63842d., pid=51, masterSystemTime=1731954261502 2024-11-18T18:24:22,077 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731954260238.c3e2f3fb6516fcaaef4d0a2c8b63842d. 2024-11-18T18:24:22,077 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731954260238.c3e2f3fb6516fcaaef4d0a2c8b63842d. 2024-11-18T18:24:22,078 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731954260238.40a213dc6b2a0fd809c71612c437ab83. 2024-11-18T18:24:22,078 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => 40a213dc6b2a0fd809c71612c437ab83, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731954260238.40a213dc6b2a0fd809c71612c437ab83.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-18T18:24:22,078 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 40a213dc6b2a0fd809c71612c437ab83 2024-11-18T18:24:22,078 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731954260238.40a213dc6b2a0fd809c71612c437ab83.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:24:22,084 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for 40a213dc6b2a0fd809c71612c437ab83 2024-11-18T18:24:22,084 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for 40a213dc6b2a0fd809c71612c437ab83 2024-11-18T18:24:22,085 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=c3e2f3fb6516fcaaef4d0a2c8b63842d, regionState=OPEN, openSeqNum=2, regionLocation=961a81da9827,36379,1731954257826 2024-11-18T18:24:22,091 INFO [StoreOpener-40a213dc6b2a0fd809c71612c437ab83-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 40a213dc6b2a0fd809c71612c437ab83 2024-11-18T18:24:22,094 INFO [StoreOpener-40a213dc6b2a0fd809c71612c437ab83-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40a213dc6b2a0fd809c71612c437ab83 columnFamilyName colfamily11 2024-11-18T18:24:22,094 DEBUG [StoreOpener-40a213dc6b2a0fd809c71612c437ab83-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:22,095 INFO [StoreOpener-40a213dc6b2a0fd809c71612c437ab83-1 {}] regionserver.HStore(327): Store=40a213dc6b2a0fd809c71612c437ab83/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:22,095 INFO [StoreOpener-40a213dc6b2a0fd809c71612c437ab83-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 40a213dc6b2a0fd809c71612c437ab83 2024-11-18T18:24:22,097 INFO [StoreOpener-40a213dc6b2a0fd809c71612c437ab83-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40a213dc6b2a0fd809c71612c437ab83 columnFamilyName colfamily21 2024-11-18T18:24:22,097 DEBUG [StoreOpener-40a213dc6b2a0fd809c71612c437ab83-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:22,098 INFO [StoreOpener-40a213dc6b2a0fd809c71612c437ab83-1 {}] regionserver.HStore(327): Store=40a213dc6b2a0fd809c71612c437ab83/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:22,098 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure c3e2f3fb6516fcaaef4d0a2c8b63842d, server=961a81da9827,36379,1731954257826 because future has completed 2024-11-18T18:24:22,099 INFO [StoreOpener-40a213dc6b2a0fd809c71612c437ab83-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 40a213dc6b2a0fd809c71612c437ab83 2024-11-18T18:24:22,100 INFO [StoreOpener-40a213dc6b2a0fd809c71612c437ab83-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40a213dc6b2a0fd809c71612c437ab83 columnFamilyName colfamily31 2024-11-18T18:24:22,100 DEBUG [StoreOpener-40a213dc6b2a0fd809c71612c437ab83-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:24:22,101 INFO [StoreOpener-40a213dc6b2a0fd809c71612c437ab83-1 {}] regionserver.HStore(327): Store=40a213dc6b2a0fd809c71612c437ab83/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:24:22,101 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for 40a213dc6b2a0fd809c71612c437ab83 2024-11-18T18:24:22,102 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/40a213dc6b2a0fd809c71612c437ab83 2024-11-18T18:24:22,103 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/40a213dc6b2a0fd809c71612c437ab83 2024-11-18T18:24:22,105 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for 40a213dc6b2a0fd809c71612c437ab83 2024-11-18T18:24:22,106 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for 40a213dc6b2a0fd809c71612c437ab83 2024-11-18T18:24:22,107 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T18:24:22,108 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=15 2024-11-18T18:24:22,108 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure c3e2f3fb6516fcaaef4d0a2c8b63842d, server=961a81da9827,36379,1731954257826 in 729 msec 2024-11-18T18:24:22,109 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for 40a213dc6b2a0fd809c71612c437ab83 2024-11-18T18:24:22,114 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c3e2f3fb6516fcaaef4d0a2c8b63842d, ASSIGN in 1.0420 sec 2024-11-18T18:24:22,117 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/40a213dc6b2a0fd809c71612c437ab83/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T18:24:22,117 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened 40a213dc6b2a0fd809c71612c437ab83; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69683422, jitterRate=0.038363903760910034}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T18:24:22,117 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 40a213dc6b2a0fd809c71612c437ab83 2024-11-18T18:24:22,118 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for 40a213dc6b2a0fd809c71612c437ab83: Running coprocessor pre-open hook at 1731954262085Writing region info on filesystem at 1731954262085Initializing all the Stores at 1731954262087 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954262087Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954262091 (+4 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954262091Cleaning up temporary data from old regions at 1731954262106 (+15 ms)Running coprocessor post-open hooks at 1731954262117 (+11 ms)Region opened successfully at 1731954262118 (+1 ms) 2024-11-18T18:24:22,132 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731954260238.40a213dc6b2a0fd809c71612c437ab83., pid=50, masterSystemTime=1731954261502 2024-11-18T18:24:22,145 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=40a213dc6b2a0fd809c71612c437ab83, regionState=OPEN, openSeqNum=2, regionLocation=961a81da9827,36379,1731954257826 2024-11-18T18:24:22,144 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731954260238.40a213dc6b2a0fd809c71612c437ab83. 2024-11-18T18:24:22,147 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731954260238.40a213dc6b2a0fd809c71612c437ab83. 2024-11-18T18:24:22,150 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 40a213dc6b2a0fd809c71612c437ab83, server=961a81da9827,36379,1731954257826 because future has completed 2024-11-18T18:24:22,156 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=12 2024-11-18T18:24:22,156 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 40a213dc6b2a0fd809c71612c437ab83, server=961a81da9827,36379,1731954257826 in 782 msec 2024-11-18T18:24:22,161 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=4 2024-11-18T18:24:22,161 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=40a213dc6b2a0fd809c71612c437ab83, ASSIGN in 1.0900 sec 2024-11-18T18:24:22,163 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T18:24:22,164 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731954262163"}]},"ts":"1731954262163"} 2024-11-18T18:24:22,167 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, state=ENABLED in hbase:meta 2024-11-18T18:24:22,169 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T18:24:22,176 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true in 1.9210 sec 2024-11-18T18:24:22,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T18:24:22,415 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true completed 2024-11-18T18:24:22,415 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true get assigned. Timeout = 60000ms 2024-11-18T18:24:22,417 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T18:24:22,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36379 {}] regionserver.StoreScanner(1138): Switch to stream read (scanned=32802 bytes) of info 2024-11-18T18:24:22,454 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true assigned to meta. Checking AM states. 2024-11-18T18:24:22,455 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T18:24:22,456 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true assigned. 2024-11-18T18:24:22,519 DEBUG [Time-limited test {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aab', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731954260238.8bb7ae0747eda2290799ca8fa83f9ede., hostname=961a81da9827,38709,1731954257723, seqNum=2] 2024-11-18T18:24:22,519 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aaa', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731954260238.8bb7ae0747eda2290799ca8fa83f9ede., hostname=961a81da9827,38709,1731954257723, seqNum=2] 2024-11-18T18:24:22,609 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='bbc', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731954260238.22971deb0db38b7cccde5c3bc0868f22., hostname=961a81da9827,42895,1731954257867, seqNum=2] 2024-11-18T18:24:22,611 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='bbd', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731954260238.22971deb0db38b7cccde5c3bc0868f22., hostname=961a81da9827,42895,1731954257867, seqNum=2] 2024-11-18T18:24:22,612 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='bbe', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731954260238.22971deb0db38b7cccde5c3bc0868f22., hostname=961a81da9827,42895,1731954257867, seqNum=2] 2024-11-18T18:24:22,614 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='bbb', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731954260238.22971deb0db38b7cccde5c3bc0868f22., hostname=961a81da9827,42895,1731954257867, seqNum=2] 2024-11-18T18:24:22,616 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='bbf', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731954260238.22971deb0db38b7cccde5c3bc0868f22., hostname=961a81da9827,42895,1731954257867, seqNum=2] 2024-11-18T18:24:22,687 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ccc', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731954260238.43138a6a12c2efea7093064622bb3cd8., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:22,690 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ccd', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731954260238.43138a6a12c2efea7093064622bb3cd8., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:22,691 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='cce', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731954260238.43138a6a12c2efea7093064622bb3cd8., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:22,693 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ccf', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731954260238.43138a6a12c2efea7093064622bb3cd8., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:22,694 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='cch', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731954260238.43138a6a12c2efea7093064622bb3cd8., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:22,696 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='cci', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731954260238.43138a6a12c2efea7093064622bb3cd8., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:22,698 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ccj', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731954260238.43138a6a12c2efea7093064622bb3cd8., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:22,699 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ccg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731954260238.43138a6a12c2efea7093064622bb3cd8., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:22,748 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddd', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731954260238.bb099f0975ebdb618ef8e3b2c263bbca., hostname=961a81da9827,42895,1731954257867, seqNum=2] 2024-11-18T18:24:22,751 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='dde', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731954260238.bb099f0975ebdb618ef8e3b2c263bbca., hostname=961a81da9827,42895,1731954257867, seqNum=2] 2024-11-18T18:24:22,756 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731954260238.bb099f0975ebdb618ef8e3b2c263bbca., hostname=961a81da9827,42895,1731954257867, seqNum=2] 2024-11-18T18:24:22,770 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddf', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731954260238.bb099f0975ebdb618ef8e3b2c263bbca., hostname=961a81da9827,42895,1731954257867, seqNum=2] 2024-11-18T18:24:22,772 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddh', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731954260238.bb099f0975ebdb618ef8e3b2c263bbca., hostname=961a81da9827,42895,1731954257867, seqNum=2] 2024-11-18T18:24:22,782 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddj', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731954260238.bb099f0975ebdb618ef8e3b2c263bbca., hostname=961a81da9827,42895,1731954257867, seqNum=2] 2024-11-18T18:24:22,784 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddi', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731954260238.bb099f0975ebdb618ef8e3b2c263bbca., hostname=961a81da9827,42895,1731954257867, seqNum=2] 2024-11-18T18:24:22,805 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eee', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731954260238.bcf8695060a326b7d24615ba6cb932b1., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:22,807 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eef', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731954260238.bcf8695060a326b7d24615ba6cb932b1., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:22,809 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eeg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731954260238.bcf8695060a326b7d24615ba6cb932b1., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:22,820 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eeh', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731954260238.bcf8695060a326b7d24615ba6cb932b1., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:22,878 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='fff', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731954260238.fb91ff28ef3c18f0dc0d3f26aac68eca., hostname=961a81da9827,38709,1731954257723, seqNum=2] 2024-11-18T18:24:22,894 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ffg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731954260238.fb91ff28ef3c18f0dc0d3f26aac68eca., hostname=961a81da9827,38709,1731954257723, seqNum=2] 2024-11-18T18:24:22,897 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ffh', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731954260238.fb91ff28ef3c18f0dc0d3f26aac68eca., hostname=961a81da9827,38709,1731954257723, seqNum=2] 2024-11-18T18:24:22,899 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ffi', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731954260238.fb91ff28ef3c18f0dc0d3f26aac68eca., hostname=961a81da9827,38709,1731954257723, seqNum=2] 2024-11-18T18:24:22,901 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ffj', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731954260238.fb91ff28ef3c18f0dc0d3f26aac68eca., hostname=961a81da9827,38709,1731954257723, seqNum=2] 2024-11-18T18:24:22,906 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ffk', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731954260238.fb91ff28ef3c18f0dc0d3f26aac68eca., hostname=961a81da9827,38709,1731954257723, seqNum=2] 2024-11-18T18:24:22,969 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggh', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731954260238.c8b4dd2a6d8dd59085c547b55aac0de6., hostname=961a81da9827,42895,1731954257867, seqNum=2] 2024-11-18T18:24:22,973 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggi', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731954260238.c8b4dd2a6d8dd59085c547b55aac0de6., hostname=961a81da9827,42895,1731954257867, seqNum=2] 2024-11-18T18:24:22,975 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggj', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731954260238.c8b4dd2a6d8dd59085c547b55aac0de6., hostname=961a81da9827,42895,1731954257867, seqNum=2] 2024-11-18T18:24:22,977 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731954260238.c8b4dd2a6d8dd59085c547b55aac0de6., hostname=961a81da9827,42895,1731954257867, seqNum=2] 2024-11-18T18:24:22,979 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggk', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731954260238.c8b4dd2a6d8dd59085c547b55aac0de6., hostname=961a81da9827,42895,1731954257867, seqNum=2] 2024-11-18T18:24:22,980 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggl', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731954260238.c8b4dd2a6d8dd59085c547b55aac0de6., hostname=961a81da9827,42895,1731954257867, seqNum=2] 2024-11-18T18:24:22,982 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731954260238.c8b4dd2a6d8dd59085c547b55aac0de6., hostname=961a81da9827,42895,1731954257867, seqNum=2] 2024-11-18T18:24:22,984 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggn', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731954260238.c8b4dd2a6d8dd59085c547b55aac0de6., hostname=961a81da9827,42895,1731954257867, seqNum=2] 2024-11-18T18:24:23,023 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhi', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731954260238.40a213dc6b2a0fd809c71612c437ab83., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:23,030 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhh', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731954260238.40a213dc6b2a0fd809c71612c437ab83., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:23,034 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhk', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731954260238.40a213dc6b2a0fd809c71612c437ab83., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:23,036 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhj', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731954260238.40a213dc6b2a0fd809c71612c437ab83., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:23,038 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731954260238.40a213dc6b2a0fd809c71612c437ab83., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:23,040 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhl', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731954260238.40a213dc6b2a0fd809c71612c437ab83., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:23,041 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhn', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731954260238.40a213dc6b2a0fd809c71612c437ab83., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:23,043 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hho', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731954260238.40a213dc6b2a0fd809c71612c437ab83., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:23,099 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iii', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731954260238.f39571a93e25091090ea0831d54936b5., hostname=961a81da9827,38709,1731954257723, seqNum=2] 2024-11-18T18:24:23,102 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iij', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731954260238.f39571a93e25091090ea0831d54936b5., hostname=961a81da9827,38709,1731954257723, seqNum=2] 2024-11-18T18:24:23,104 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iik', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731954260238.f39571a93e25091090ea0831d54936b5., hostname=961a81da9827,38709,1731954257723, seqNum=2] 2024-11-18T18:24:23,110 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iil', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731954260238.f39571a93e25091090ea0831d54936b5., hostname=961a81da9827,38709,1731954257723, seqNum=2] 2024-11-18T18:24:23,173 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjk', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731954260238.97a938e093c14701871e85a1fbb6e07d., hostname=961a81da9827,42895,1731954257867, seqNum=2] 2024-11-18T18:24:23,176 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjj', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731954260238.97a938e093c14701871e85a1fbb6e07d., hostname=961a81da9827,42895,1731954257867, seqNum=2] 2024-11-18T18:24:23,178 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjl', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731954260238.97a938e093c14701871e85a1fbb6e07d., hostname=961a81da9827,42895,1731954257867, seqNum=2] 2024-11-18T18:24:23,180 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731954260238.97a938e093c14701871e85a1fbb6e07d., hostname=961a81da9827,42895,1731954257867, seqNum=2] 2024-11-18T18:24:23,182 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjn', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731954260238.97a938e093c14701871e85a1fbb6e07d., hostname=961a81da9827,42895,1731954257867, seqNum=2] 2024-11-18T18:24:23,185 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjo', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731954260238.97a938e093c14701871e85a1fbb6e07d., hostname=961a81da9827,42895,1731954257867, seqNum=2] 2024-11-18T18:24:23,186 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731954260238.97a938e093c14701871e85a1fbb6e07d., hostname=961a81da9827,42895,1731954257867, seqNum=2] 2024-11-18T18:24:23,188 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731954260238.97a938e093c14701871e85a1fbb6e07d., hostname=961a81da9827,42895,1731954257867, seqNum=2] 2024-11-18T18:24:23,223 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkk', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731954260238.c3e2f3fb6516fcaaef4d0a2c8b63842d., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:23,227 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkl', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731954260238.c3e2f3fb6516fcaaef4d0a2c8b63842d., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:23,230 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731954260238.c3e2f3fb6516fcaaef4d0a2c8b63842d., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:23,232 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkn', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731954260238.c3e2f3fb6516fcaaef4d0a2c8b63842d., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:23,235 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kko', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731954260238.c3e2f3fb6516fcaaef4d0a2c8b63842d., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:23,245 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731954260238.c3e2f3fb6516fcaaef4d0a2c8b63842d., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:23,248 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731954260238.c3e2f3fb6516fcaaef4d0a2c8b63842d., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:23,251 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkr', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731954260238.c3e2f3fb6516fcaaef4d0a2c8b63842d., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:23,298 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='llm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731954260238.97377a91d35f8b242120f603c456c41e., hostname=961a81da9827,38709,1731954257723, seqNum=2] 2024-11-18T18:24:23,300 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='lln', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731954260238.97377a91d35f8b242120f603c456c41e., hostname=961a81da9827,38709,1731954257723, seqNum=2] 2024-11-18T18:24:23,306 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='llo', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731954260238.97377a91d35f8b242120f603c456c41e., hostname=961a81da9827,38709,1731954257723, seqNum=2] 2024-11-18T18:24:23,309 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='llp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731954260238.97377a91d35f8b242120f603c456c41e., hostname=961a81da9827,38709,1731954257723, seqNum=2] 2024-11-18T18:24:23,311 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='llq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731954260238.97377a91d35f8b242120f603c456c41e., hostname=961a81da9827,38709,1731954257723, seqNum=2] 2024-11-18T18:24:23,313 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='llr', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731954260238.97377a91d35f8b242120f603c456c41e., hostname=961a81da9827,38709,1731954257723, seqNum=2] 2024-11-18T18:24:23,318 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='lll', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731954260238.97377a91d35f8b242120f603c456c41e., hostname=961a81da9827,38709,1731954257723, seqNum=2] 2024-11-18T18:24:23,324 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='lls', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731954260238.97377a91d35f8b242120f603c456c41e., hostname=961a81da9827,38709,1731954257723, seqNum=2] 2024-11-18T18:24:23,384 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731954260238.a244d0ab318eb4dd77bc5235828d9d51., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:23,395 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmo', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731954260238.a244d0ab318eb4dd77bc5235828d9d51., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:23,397 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmn', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731954260238.a244d0ab318eb4dd77bc5235828d9d51., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:23,400 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731954260238.a244d0ab318eb4dd77bc5235828d9d51., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:23,402 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731954260238.a244d0ab318eb4dd77bc5235828d9d51., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:23,462 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nnn', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731954260238.012c277c99c564dd90e8e5f6850b6ce0., hostname=961a81da9827,42895,1731954257867, seqNum=2] 2024-11-18T18:24:23,465 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nno', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731954260238.012c277c99c564dd90e8e5f6850b6ce0., hostname=961a81da9827,42895,1731954257867, seqNum=2] 2024-11-18T18:24:23,467 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nnp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731954260238.012c277c99c564dd90e8e5f6850b6ce0., hostname=961a81da9827,42895,1731954257867, seqNum=2] 2024-11-18T18:24:23,468 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nnq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731954260238.012c277c99c564dd90e8e5f6850b6ce0., hostname=961a81da9827,42895,1731954257867, seqNum=2] 2024-11-18T18:24:23,518 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ooo', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731954260238.140f5077d3b730b643b558ebe8a13abd., hostname=961a81da9827,38709,1731954257723, seqNum=2] 2024-11-18T18:24:23,520 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='oop', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731954260238.140f5077d3b730b643b558ebe8a13abd., hostname=961a81da9827,38709,1731954257723, seqNum=2] 2024-11-18T18:24:23,522 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ooq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731954260238.140f5077d3b730b643b558ebe8a13abd., hostname=961a81da9827,38709,1731954257723, seqNum=2] 2024-11-18T18:24:23,524 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='oor', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731954260238.140f5077d3b730b643b558ebe8a13abd., hostname=961a81da9827,38709,1731954257723, seqNum=2] 2024-11-18T18:24:23,571 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731954260238.5e69ef2147d87d1b83dbcbb4705cbabe., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:23,573 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731954260238.5e69ef2147d87d1b83dbcbb4705cbabe., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:23,575 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppr', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731954260238.5e69ef2147d87d1b83dbcbb4705cbabe., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:23,577 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='pps', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731954260238.5e69ef2147d87d1b83dbcbb4705cbabe., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:23,578 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppt', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731954260238.5e69ef2147d87d1b83dbcbb4705cbabe., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:23,579 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppu', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731954260238.5e69ef2147d87d1b83dbcbb4705cbabe., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:23,630 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqr', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731954260238.fb8fa4293ea5e58c18dbc5b978355ca9., hostname=961a81da9827,42895,1731954257867, seqNum=2] 2024-11-18T18:24:23,631 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731954260238.fb8fa4293ea5e58c18dbc5b978355ca9., hostname=961a81da9827,42895,1731954257867, seqNum=2] 2024-11-18T18:24:23,633 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqs', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731954260238.fb8fa4293ea5e58c18dbc5b978355ca9., hostname=961a81da9827,42895,1731954257867, seqNum=2] 2024-11-18T18:24:23,635 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqt', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731954260238.fb8fa4293ea5e58c18dbc5b978355ca9., hostname=961a81da9827,42895,1731954257867, seqNum=2] 2024-11-18T18:24:23,637 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqu', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731954260238.fb8fa4293ea5e58c18dbc5b978355ca9., hostname=961a81da9827,42895,1731954257867, seqNum=2] 2024-11-18T18:24:23,638 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731954260238.fb8fa4293ea5e58c18dbc5b978355ca9., hostname=961a81da9827,42895,1731954257867, seqNum=2] 2024-11-18T18:24:23,675 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rrs', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731954260238.69ae79c6f9eeed559fa88701f773706c., hostname=961a81da9827,38709,1731954257723, seqNum=2] 2024-11-18T18:24:23,677 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rrr', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731954260238.69ae79c6f9eeed559fa88701f773706c., hostname=961a81da9827,38709,1731954257723, seqNum=2] 2024-11-18T18:24:23,679 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rrt', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731954260238.69ae79c6f9eeed559fa88701f773706c., hostname=961a81da9827,38709,1731954257723, seqNum=2] 2024-11-18T18:24:23,680 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rru', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731954260238.69ae79c6f9eeed559fa88701f773706c., hostname=961a81da9827,38709,1731954257723, seqNum=2] 2024-11-18T18:24:23,682 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rrv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731954260238.69ae79c6f9eeed559fa88701f773706c., hostname=961a81da9827,38709,1731954257723, seqNum=2] 2024-11-18T18:24:23,722 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='sss', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731954260238.75a13ed51395b1016b90c6b0d90a7d0a., hostname=961a81da9827,42895,1731954257867, seqNum=2] 2024-11-18T18:24:23,725 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='sst', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731954260238.75a13ed51395b1016b90c6b0d90a7d0a., hostname=961a81da9827,42895,1731954257867, seqNum=2] 2024-11-18T18:24:23,726 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ssv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731954260238.75a13ed51395b1016b90c6b0d90a7d0a., hostname=961a81da9827,42895,1731954257867, seqNum=2] 2024-11-18T18:24:23,728 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ssu', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731954260238.75a13ed51395b1016b90c6b0d90a7d0a., hostname=961a81da9827,42895,1731954257867, seqNum=2] 2024-11-18T18:24:23,730 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ssx', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731954260238.75a13ed51395b1016b90c6b0d90a7d0a., hostname=961a81da9827,42895,1731954257867, seqNum=2] 2024-11-18T18:24:23,732 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ssw', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731954260238.75a13ed51395b1016b90c6b0d90a7d0a., hostname=961a81da9827,42895,1731954257867, seqNum=2] 2024-11-18T18:24:23,772 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ttt', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731954260238.aaf2435418703b2a31c1190af1560ba8., hostname=961a81da9827,42895,1731954257867, seqNum=2] 2024-11-18T18:24:23,775 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ttu', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731954260238.aaf2435418703b2a31c1190af1560ba8., hostname=961a81da9827,42895,1731954257867, seqNum=2] 2024-11-18T18:24:23,777 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ttv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731954260238.aaf2435418703b2a31c1190af1560ba8., hostname=961a81da9827,42895,1731954257867, seqNum=2] 2024-11-18T18:24:23,779 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ttw', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731954260238.aaf2435418703b2a31c1190af1560ba8., hostname=961a81da9827,42895,1731954257867, seqNum=2] 2024-11-18T18:24:23,781 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ttx', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731954260238.aaf2435418703b2a31c1190af1560ba8., hostname=961a81da9827,42895,1731954257867, seqNum=2] 2024-11-18T18:24:23,817 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uuu', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731954260238.d1ad74f79ba4d7fd1619c83a55106331., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:23,820 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uuv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731954260238.d1ad74f79ba4d7fd1619c83a55106331., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:23,821 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uuw', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731954260238.d1ad74f79ba4d7fd1619c83a55106331., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:23,823 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uuy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731954260238.d1ad74f79ba4d7fd1619c83a55106331., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:23,825 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uux', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731954260238.d1ad74f79ba4d7fd1619c83a55106331., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:23,828 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uuz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731954260238.d1ad74f79ba4d7fd1619c83a55106331., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:23,830 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uva', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731954260238.d1ad74f79ba4d7fd1619c83a55106331., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:23,887 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731954260238.81f0db1bea1e11879dc1cafb06c0756d., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:23,889 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvw', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731954260238.81f0db1bea1e11879dc1cafb06c0756d., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:23,891 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvx', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731954260238.81f0db1bea1e11879dc1cafb06c0756d., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:23,893 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731954260238.81f0db1bea1e11879dc1cafb06c0756d., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:23,895 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731954260238.81f0db1bea1e11879dc1cafb06c0756d., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:23,951 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='www', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731954260238.2a8fcaf8905dcde0537806fb16b9ff4c., hostname=961a81da9827,38709,1731954257723, seqNum=2] 2024-11-18T18:24:23,953 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='wwx', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731954260238.2a8fcaf8905dcde0537806fb16b9ff4c., hostname=961a81da9827,38709,1731954257723, seqNum=2] 2024-11-18T18:24:23,954 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='wwz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731954260238.2a8fcaf8905dcde0537806fb16b9ff4c., hostname=961a81da9827,38709,1731954257723, seqNum=2] 2024-11-18T18:24:23,956 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='wwy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731954260238.2a8fcaf8905dcde0537806fb16b9ff4c., hostname=961a81da9827,38709,1731954257723, seqNum=2] 2024-11-18T18:24:23,960 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='wxb', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731954260238.2a8fcaf8905dcde0537806fb16b9ff4c., hostname=961a81da9827,38709,1731954257723, seqNum=2] 2024-11-18T18:24:23,962 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='wxa', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731954260238.2a8fcaf8905dcde0537806fb16b9ff4c., hostname=961a81da9827,38709,1731954257723, seqNum=2] 2024-11-18T18:24:23,998 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xxx', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731954260238.579ee42a94597fdc1db9a4820a53ee64., hostname=961a81da9827,42895,1731954257867, seqNum=2] 2024-11-18T18:24:24,000 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xxy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731954260238.579ee42a94597fdc1db9a4820a53ee64., hostname=961a81da9827,42895,1731954257867, seqNum=2] 2024-11-18T18:24:24,002 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xxz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731954260238.579ee42a94597fdc1db9a4820a53ee64., hostname=961a81da9827,42895,1731954257867, seqNum=2] 2024-11-18T18:24:24,004 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xya', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731954260238.579ee42a94597fdc1db9a4820a53ee64., hostname=961a81da9827,42895,1731954257867, seqNum=2] 2024-11-18T18:24:24,006 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xyb', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731954260238.579ee42a94597fdc1db9a4820a53ee64., hostname=961a81da9827,42895,1731954257867, seqNum=2] 2024-11-18T18:24:24,045 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yyy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731954260238.779798cd60db955ed78a4329cac08969., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:24,048 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yyz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731954260238.779798cd60db955ed78a4329cac08969., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:24,049 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yza', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731954260238.779798cd60db955ed78a4329cac08969., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:24,052 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yzb', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731954260238.779798cd60db955ed78a4329cac08969., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:24,054 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yzc', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731954260238.779798cd60db955ed78a4329cac08969., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:24,055 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yzd', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731954260238.779798cd60db955ed78a4329cac08969., hostname=961a81da9827,36379,1731954257826, seqNum=2] 2024-11-18T18:24:24,093 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='zzz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731954260238.08ccc32db22fb75f5466ad774ad9e8ba., hostname=961a81da9827,38709,1731954257723, seqNum=2] 2024-11-18T18:24:24,152 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T18:24:24,171 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T18:24:24,173 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59532, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T18:24:24,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36379 {}] regionserver.RSRpcServices(2610): Large batch operation detected (greater than 5000) (HBASE-18023). Requested Number of Rows: 6327 Client: jenkins//172.17.0.2 first region in multi=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731954260238.43138a6a12c2efea7093064622bb3cd8. 2024-11-18T18:24:24,194 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37354, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T18:24:24,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.RSRpcServices(2610): Large batch operation detected (greater than 5000) (HBASE-18023). Requested Number of Rows: 6327 Client: jenkins//172.17.0.2 first region in multi=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731954260238.22971deb0db38b7cccde5c3bc0868f22. 2024-11-18T18:24:25,140 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,, stopping at row=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true ,, for max=2147483647 with caching=100 2024-11-18T18:24:25,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36379 {}] regionserver.StoreScanner(1138): Switch to stream read (scanned=32802 bytes) of info 2024-11-18T18:24:25,337 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T18:24:25,389 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true' 2024-11-18T18:24:25,392 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-18T18:24:25,401 WARN [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: true 2024-11-18T18:24:25,402 INFO [Time-limited test {}] assignment.AssignmentTestingUtil(82): KILL REGION SERVER 961a81da9827,36379,1731954257826 2024-11-18T18:24:25,402 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing 961a81da9827,36379,1731954257826 2024-11-18T18:24:25,403 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 961a81da9827,36379,1731954257826: Simulated kill ***** 2024-11-18T18:24:25,403 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-18T18:24:25,406 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-18T18:24:25,408 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-18T18:24:25,411 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-18T18:24:25,411 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-18T18:24:25,421 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 615318784 }, "NonHeapMemoryUsage": { "committed": 157941760, "init": 7667712, "max": -1, "used": 155485848 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "961a81da9827", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 2, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 1513440, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 359, "ProcessCallTime_min": 0, "ProcessCallTime_max": 584, "ProcessCallTime_mean": 6, "ProcessCallTime_25th_percentile": 146, "ProcessCallTime_median": 292, "ProcessCallTime_75th_percentile": 438, "ProcessCallTime_90th_percentile": 525, "ProcessCallTime_95th_percentile": 554, "ProcessCallTime_98th_percentile": 572, "ProcessCallTime_99th_percentile": 578, "ProcessCallTime_99.9th_percentile": 583, "ProcessCallTime_TimeRangeCount_0-1": 359, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 359, "QueueCallTime_min": 0, "QueueCallTime_max": 24, "QueueCallTime_mean": 1, "QueueCallTime_25th_percentile": 6, "QueueCallTime_median": 12, "QueueCallTime_75th_percentile": 18, "QueueCallTime_90th_percentile": 21, "QueueCallTime_95th_percentile": 22, "QueueCallTime_98th_percentile": 23, "QueueCallTime_99th_percentile": 23, "QueueCallTime_99.9th_percentile": 23, "QueueCallTime_TimeRangeCount_0-1": 359, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 359, "TotalCallTime_min": 0, "TotalCallTime_max": 584, "TotalCallTime_mean": 8, "TotalCallTime_25th_percentile": 146, "TotalCallTime_median": 292, "TotalCallTime_75th_percentile": 438, "TotalCallTime_90th_percentile": 525, "TotalCallTime_95th_percentile": 554, "TotalCallTime_98th_percentile": 572, "TotalCallTime_99th_percentile": 578, "TotalCallTime_99.9th_percentile": 583, "TotalCallTime_TimeRangeCount_0-1": 359, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 359, "ResponseSize_min": 0, "ResponseSize_max": 59075, "ResponseSize_mean": 13297, "ResponseSize_25th_percentile": 14768, "ResponseSize_median": 29537, "ResponseSize_75th_percentile": 44306, "ResponseSize_90th_percentile": 53167, "ResponseSize_95th_percentile": 56121, "ResponseSize_98th_percentile": 57893, "ResponseSize_99th_percentile": 58484, "ResponseSize_99.9th_percentile": 59015, "ResponseSize_SizeRangeCount_0-10": 359, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 359, "RequestSize_min": 42, "RequestSize_max": 508072, "RequestSize_mean": 4203, "RequestSize_25th_percentile": 127049, "RequestSize_median": 254057, "RequestSize_75th_percentile": 381064, "RequestSize_90th_percentile": 457269, "RequestSize_95th_percentile": 482670, "RequestSize_98th_percentile": 497911, "RequestSize_99th_percentile": 502991, "RequestSize_99.9th_percentile": 507563, "RequestSize_SizeRangeCount_0-10": 359, "sentBytes": 3547628 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "961a81da9827", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:59292", "tag.serverName": "961a81da9827,36379,1731954257826", "tag.clusterId": "d961f11e-5c15-416e-a960-3926a0e53aca", "tag.Context": "regionserver", "tag.Hostname": "961a81da9827", "regionCount": 10, "storeCount": 31, "hlogFileCount": 2, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 50772, "memStoreHeapSize": 78576, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1731954257826, "averageRegionSize": 5077, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 0, "localBytesRead": 0, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 244.2, "writeRequestRatePerSecond": 16.400000000000002, "ByteBuffAllocatorHeapAllocationBytes": 0, "ByteBuffAllocatorPoolAllocationBytes": 11448320, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 5, "activeScanners": 0, "totalRequestCount": 319, "totalRowActionRequestCount": 1303, "readRequestCount": 1221, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 82, "rpcGetRequestCount": 1, "rpcFullScanRequestCount": 18, "rpcScanRequestCount": 260, "rpcMultiRequestCount": 2, "rpcMutateRequestCount": 56, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 173, "ScanTime_min": 0, "ScanTime_max": 16, "ScanTime_mean": 1, "ScanTime_25th_percentile": 4, "ScanTime_median": 8, "ScanTime_75th_percentile": 12, "ScanTime_90th_percentile": 14, "ScanTime_95th_percentile": 15, "ScanTime_98th_percentile": 15, "ScanTime_99th_percentile": 15, "ScanTime_99.9th_percentile": 15, "ScanTime_TimeRangeCount_0-1": 173, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 56, "Put_min": 1, "Put_max": 10, "Put_mean": 2, "Put_25th_percentile": 3, "Put_median": 5, "Put_75th_percentile": 7, "Put_90th_percentile": 9, "Put_95th_percentile": 9, "Put_98th_percentile": 9, "Put_99th_percentile": 9, "Put_99.9th_percentile": 9, "Put_TimeRangeCount_0-1": 56, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 10, "PutBatch_min": 14, "PutBatch_max": 98, "PutBatch_mean": 58, "PutBatch_25th_percentile": 35, "PutBatch_median": 56, "PutBatch_75th_percentile": 77, "PutBatch_90th_percentile": 89, "PutBatch_95th_percentile": 93, "PutBatch_98th_percentile": 96, "PutBatch_99th_percentile": 97, "PutBatch_99.9th_percentile": 97, "PutBatch_TimeRangeCount_0-1": 10, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 1, "Get_min": 2, "Get_max": 2, "Get_mean": 2, "Get_25th_percentile": 2, "Get_median": 2, "Get_75th_percentile": 2, "Get_90th_percentile": 2, "Get_95th_percentile": 2, "Get_98th_percentile": 2, "Get_99th_percentile": 2, "Get_99.9th_percentile": 2, "Get_TimeRangeCount_0-1": 1, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 173, "ScanSize_min": 0, "ScanSize_max": 56240, "ScanSize_mean": 15592, "ScanSize_25th_percentile": 14060, "ScanSize_median": 28120, "ScanSize_75th_percentile": 42180, "ScanSize_90th_percentile": 50616, "ScanSize_95th_percentile": 53428, "ScanSize_98th_percentile": 55115, "ScanSize_99th_percentile": 55677, "ScanSize_99.9th_percentile": 56183, "ScanSize_SizeRangeCount_0-10": 173, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-18T18:24:25,426 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33089 {}] master.MasterRpcServices(700): 961a81da9827,36379,1731954257826 reported a fatal error: ***** ABORTING region server 961a81da9827,36379,1731954257826: Simulated kill ***** 2024-11-18T18:24:25,428 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '961a81da9827,36379,1731954257826' ***** 2024-11-18T18:24:25,428 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-18T18:24:25,429 INFO [RS:1;961a81da9827:36379 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-18T18:24:25,429 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T18:24:25,429 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-18T18:24:25,430 INFO [RS:1;961a81da9827:36379 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-18T18:24:25,430 INFO [RS:1;961a81da9827:36379 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-18T18:24:25,430 INFO [RS:1;961a81da9827:36379 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T18:24:25,431 INFO [RS:1;961a81da9827:36379 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;961a81da9827:36379. 2024-11-18T18:24:25,431 DEBUG [RS:1;961a81da9827:36379 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T18:24:25,438 DEBUG [RS:1;961a81da9827:36379 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T18:24:25,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741836_1012 (size=54214) 2024-11-18T18:24:25,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741836_1012 (size=54214) 2024-11-18T18:24:25,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741836_1012 (size=54214) 2024-11-18T18:24:25,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741833_1009 (size=334920) 2024-11-18T18:24:25,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741833_1009 (size=334920) 2024-11-18T18:24:25,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741833_1009 (size=334920) 2024-11-18T18:24:25,454 DEBUG [RS:1;961a81da9827:36379 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T18:24:25,454 INFO [RS:1;961a81da9827:36379 {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T18:24:25,454 INFO [RS:1;961a81da9827:36379 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36379 2024-11-18T18:24:25,458 INFO [regionserver/961a81da9827:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T18:24:25,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33089-0x10107179d9f0000, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T18:24:25,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36379-0x10107179d9f0002, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/961a81da9827,36379,1731954257826 2024-11-18T18:24:25,464 INFO [RS:1;961a81da9827:36379 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T18:24:25,466 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [961a81da9827,36379,1731954257826] 2024-11-18T18:24:25,467 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/961a81da9827,36379,1731954257826 already deleted, retry=false 2024-11-18T18:24:25,468 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 961a81da9827,36379,1731954257826 on 961a81da9827,33089,1731954257001 2024-11-18T18:24:25,475 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 961a81da9827,36379,1731954257826, splitWal=true, meta=true 2024-11-18T18:24:25,481 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 961a81da9827,36379,1731954257826, splitWal=true, meta=true 2024-11-18T18:24:25,481 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-18T18:24:25,481 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=57 for 961a81da9827,36379,1731954257826 (carryingMeta=true) 961a81da9827,36379,1731954257826/CRASHED/regionCount=10/lock=java.util.concurrent.locks.ReentrantReadWriteLock@65e7626e[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-18T18:24:25,481 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 961a81da9827,36379,1731954257826, splitWal=true, meta=true 2024-11-18T18:24:25,481 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T18:24:25,481 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 961a81da9827,36379,1731954257826, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T18:24:25,530 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T18:24:25,532 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 0 exec state=pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 961a81da9827,36379,1731954257826, splitWal=true, meta=true 2024-11-18T18:24:25,533 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-18T18:24:25,533 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T18:24:25,533 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-18T18:24:25,533 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-18T18:24:25,534 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-18T18:24:25,534 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-18T18:24:25,534 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-18T18:24:25,534 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-18T18:24:25,535 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-18T18:24:25,538 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33089 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T18:24:25,540 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-18T18:24:25,541 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-18T18:24:25,543 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-18T18:24:25,543 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 961a81da9827,36379,1731954257826, splitWal=true, meta=true 2024-11-18T18:24:25,543 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 961a81da9827,36379,1731954257826, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-18T18:24:25,547 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 11 msec 2024-11-18T18:24:25,549 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=961a81da9827,36379,1731954257826, regionLocation=961a81da9827,36379,1731954257826, openSeqNum=2 2024-11-18T18:24:25,549 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=961a81da9827,36379,1731954257826, table=hbase:meta, region=1588230740 2024-11-18T18:24:25,550 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 961a81da9827,36379,1731954257826, state=OPEN 2024-11-18T18:24:25,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42895-0x10107179d9f0003, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T18:24:25,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38709-0x10107179d9f0001, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T18:24:25,554 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:24:25,554 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:24:25,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33089-0x10107179d9f0000, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T18:24:25,555 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:24:25,566 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731954295566 2024-11-18T18:24:25,566 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-18T18:24:25,566 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 961a81da9827,36379,1731954257826, splitWal=true, meta=true 2024-11-18T18:24:25,566 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-18T18:24:25,568 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 961a81da9827:36379 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T18:24:25,568 INFO [RS:1;961a81da9827:36379 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T18:24:25,569 INFO [RS:1;961a81da9827:36379 {}] regionserver.HRegionServer(1031): Exiting; stopping=961a81da9827,36379,1731954257826; zookeeper connection closed. 2024-11-18T18:24:25,569 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36379-0x10107179d9f0002, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T18:24:25,569 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36379-0x10107179d9f0002, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T18:24:25,571 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1, error=java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:24:25,572 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-18T18:24:25,572 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 from cache 2024-11-18T18:24:25,573 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 961a81da9827:36379 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:24:25,573 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 961a81da9827,36379,1731954257826, splitWal=true, meta=true, isMeta: true 2024-11-18T18:24:25,576 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/WALs/961a81da9827,36379,1731954257826-splitting 2024-11-18T18:24:25,577 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3634edf4 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3634edf4 2024-11-18T18:24:25,577 INFO [PEWorker-1 {}] master.SplitWALManager(105): 961a81da9827,36379,1731954257826 WAL count=1, meta=true 2024-11-18T18:24:25,580 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 961a81da9827%2C36379%2C1731954257826.meta.1731954259788.meta}] 2024-11-18T18:24:25,580 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-18T18:24:25,580 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, hasLock=true; ServerCrashProcedure 961a81da9827,36379,1731954257826, splitWal=true, meta=true 2024-11-18T18:24:25,580 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T18:24:25,581 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, hasLock=true; ServerCrashProcedure 961a81da9827,36379,1731954257826, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T18:24:25,683 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T18:24:25,683 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1] 2024-11-18T18:24:25,684 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 961a81da9827:36379 this server is in the failed servers list 2024-11-18T18:24:25,685 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=961a81da9827:36379 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:24:25,685 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:24:25,685 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 from cache 2024-11-18T18:24:25,893 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T18:24:25,893 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1] 2024-11-18T18:24:25,893 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 961a81da9827:36379 this server is in the failed servers list 2024-11-18T18:24:25,893 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=961a81da9827:36379 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:24:25,893 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:24:25,894 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 from cache 2024-11-18T18:24:26,112 INFO [regionserver/961a81da9827:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-18T18:24:26,112 INFO [regionserver/961a81da9827:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-18T18:24:26,203 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T18:24:26,203 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1] 2024-11-18T18:24:26,203 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 961a81da9827:36379 this server is in the failed servers list 2024-11-18T18:24:26,204 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=961a81da9827:36379 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:24:26,204 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:24:26,204 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 from cache 2024-11-18T18:24:26,713 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T18:24:26,713 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1] 2024-11-18T18:24:26,713 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 961a81da9827:36379 this server is in the failed servers list 2024-11-18T18:24:26,713 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=961a81da9827:36379 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:24:26,714 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:24:26,714 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 from cache 2024-11-18T18:24:27,430 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-18T18:24:27,431 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-18T18:24:27,432 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-18T18:24:27,432 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true Metrics about Tables on a single HBase RegionServer 2024-11-18T18:24:27,433 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-18T18:24:27,433 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-18T18:24:27,433 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-18T18:24:27,433 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-18T18:24:27,723 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T18:24:27,723 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1] 2024-11-18T18:24:27,724 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 961a81da9827:36379 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T18:24:27,728 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 961a81da9827,36379,1731954257826 for '' in 1588230740 of hbase:meta failed, tries = 6, maxAttempts = 46, timeout = 60000 ms, time elapsed = 2158 ms java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-18T18:24:27,729 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1, error=java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:24:27,729 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-18T18:24:27,729 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 from cache 2024-11-18T18:24:27,730 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 961a81da9827:36379 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:24:29,743 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T18:24:29,743 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1] 2024-11-18T18:24:29,744 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 961a81da9827:36379 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T18:24:29,744 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 961a81da9827,36379,1731954257826 for '' in 1588230740 of hbase:meta failed, tries = 7, maxAttempts = 46, timeout = 60000 ms, time elapsed = 4177 ms java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-18T18:24:29,745 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1, error=java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:24:29,745 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-18T18:24:29,745 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 from cache 2024-11-18T18:24:29,745 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 961a81da9827:36379 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:24:33,753 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T18:24:33,753 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1] 2024-11-18T18:24:33,754 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 961a81da9827:36379 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T18:24:33,755 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 961a81da9827,36379,1731954257826 for '' in 1588230740 of hbase:meta failed, tries = 8, maxAttempts = 46, timeout = 60000 ms, time elapsed = 8188 ms java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-18T18:24:33,755 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1, error=java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:24:33,755 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-18T18:24:33,755 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 from cache 2024-11-18T18:24:33,755 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 961a81da9827:36379 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:24:43,852 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T18:24:43,853 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1] 2024-11-18T18:24:43,854 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 961a81da9827:36379 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T18:24:43,854 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 961a81da9827,36379,1731954257826 for '' in 1588230740 of hbase:meta failed, tries = 9, maxAttempts = 46, timeout = 60000 ms, time elapsed = 18287 ms java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-18T18:24:43,854 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1, error=java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:24:43,855 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-18T18:24:43,855 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 from cache 2024-11-18T18:24:43,855 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 961a81da9827:36379 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:24:46,051 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-18T18:24:52,825 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=961a81da9827:33089 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 10 more 2024-11-18T18:24:53,933 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T18:24:53,933 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1] 2024-11-18T18:24:53,934 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 961a81da9827:36379 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T18:24:53,935 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 961a81da9827,36379,1731954257826 for '' in 1588230740 of hbase:meta failed, tries = 10, maxAttempts = 46, timeout = 60000 ms, time elapsed = 28368 ms java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-18T18:24:53,935 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1, error=java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:24:53,935 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-18T18:24:53,935 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 from cache 2024-11-18T18:24:53,936 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 961a81da9827:36379 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:25:00,098 INFO [master/961a81da9827:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-18T18:25:00,098 INFO [master/961a81da9827:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-18T18:25:04,002 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T18:25:04,002 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1] 2024-11-18T18:25:04,003 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 961a81da9827:36379 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T18:25:04,004 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 961a81da9827,36379,1731954257826 for '' in 1588230740 of hbase:meta failed, tries = 11, maxAttempts = 46, timeout = 60000 ms, time elapsed = 38437 ms java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-18T18:25:04,004 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1, error=java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:25:04,004 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-18T18:25:04,004 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 from cache 2024-11-18T18:25:04,005 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 961a81da9827:36379 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:25:11,169 WARN [RS:0;961a81da9827:38709 {}] monitoring.TaskMonitor(166): Status Processing ServerCrashProcedure of 961a81da9827,36379,1731954257826: status=status unset, state=RUNNING, startTime=1731954265480, completionTime=-1 appears to have been leaked 2024-11-18T18:25:14,023 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T18:25:14,023 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1] 2024-11-18T18:25:14,024 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 961a81da9827:36379 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T18:25:14,024 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 961a81da9827,36379,1731954257826 for '' in 1588230740 of hbase:meta failed, tries = 12, maxAttempts = 46, timeout = 60000 ms, time elapsed = 48458 ms java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-18T18:25:14,025 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1, error=java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:25:14,025 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-18T18:25:14,025 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 from cache 2024-11-18T18:25:14,026 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 961a81da9827:36379 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:25:16,051 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-18T18:25:19,535 INFO [regionserver/961a81da9827:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: ExecutorStatusChore was stopped 2024-11-18T18:25:20,037 DEBUG [master/961a81da9827:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 8bb7ae0747eda2290799ca8fa83f9ede changed from -1.0 to 0.0, refreshing cache 2024-11-18T18:25:20,037 DEBUG [master/961a81da9827:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region fb91ff28ef3c18f0dc0d3f26aac68eca changed from -1.0 to 0.0, refreshing cache 2024-11-18T18:25:20,038 DEBUG [master/961a81da9827:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f39571a93e25091090ea0831d54936b5 changed from -1.0 to 0.0, refreshing cache 2024-11-18T18:25:20,038 DEBUG [master/961a81da9827:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 97377a91d35f8b242120f603c456c41e changed from -1.0 to 0.0, refreshing cache 2024-11-18T18:25:20,038 DEBUG [master/961a81da9827:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 140f5077d3b730b643b558ebe8a13abd changed from -1.0 to 0.0, refreshing cache 2024-11-18T18:25:20,038 DEBUG [master/961a81da9827:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 69ae79c6f9eeed559fa88701f773706c changed from -1.0 to 0.0, refreshing cache 2024-11-18T18:25:20,038 DEBUG [master/961a81da9827:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 2a8fcaf8905dcde0537806fb16b9ff4c changed from -1.0 to 0.0, refreshing cache 2024-11-18T18:25:20,039 DEBUG [master/961a81da9827:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 08ccc32db22fb75f5466ad774ad9e8ba changed from -1.0 to 0.0, refreshing cache 2024-11-18T18:25:20,039 DEBUG [master/961a81da9827:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 22971deb0db38b7cccde5c3bc0868f22 changed from -1.0 to 0.0, refreshing cache 2024-11-18T18:25:20,039 DEBUG [master/961a81da9827:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region bb099f0975ebdb618ef8e3b2c263bbca changed from -1.0 to 0.0, refreshing cache 2024-11-18T18:25:20,039 DEBUG [master/961a81da9827:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c8b4dd2a6d8dd59085c547b55aac0de6 changed from -1.0 to 0.0, refreshing cache 2024-11-18T18:25:20,039 DEBUG [master/961a81da9827:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 97a938e093c14701871e85a1fbb6e07d changed from -1.0 to 0.0, refreshing cache 2024-11-18T18:25:20,039 DEBUG [master/961a81da9827:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 012c277c99c564dd90e8e5f6850b6ce0 changed from -1.0 to 0.0, refreshing cache 2024-11-18T18:25:20,039 DEBUG [master/961a81da9827:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region fb8fa4293ea5e58c18dbc5b978355ca9 changed from -1.0 to 0.0, refreshing cache 2024-11-18T18:25:20,039 DEBUG [master/961a81da9827:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 75a13ed51395b1016b90c6b0d90a7d0a changed from -1.0 to 0.0, refreshing cache 2024-11-18T18:25:20,039 DEBUG [master/961a81da9827:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region aaf2435418703b2a31c1190af1560ba8 changed from -1.0 to 0.0, refreshing cache 2024-11-18T18:25:20,039 DEBUG [master/961a81da9827:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 579ee42a94597fdc1db9a4820a53ee64 changed from -1.0 to 0.0, refreshing cache 2024-11-18T18:25:25,573 WARN [Time-limited test {}] procedure.MasterProcedureTestingUtility$3(130): Failed to load meta org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=13, exceptions: 2024-11-18T18:24:25.572Z, java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:24:25.685Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=961a81da9827:36379 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:24:25.894Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=961a81da9827:36379 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:24:26.204Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=961a81da9827:36379 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:24:26.714Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=961a81da9827:36379 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:24:27.729Z, java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:24:29.745Z, java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:24:33.755Z, java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:24:43.855Z, java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:24:53.935Z, java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:25:04.004Z, java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:25:14.025Z, java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:493) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:461) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:422) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.fullScanRegions(MetaTableAccessor.java:108) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionStateStore.visitMeta(RegionStateStore.java:108) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.loadMeta(AssignmentManager.java:1907) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.joinCluster(AssignmentManager.java:1769) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:126) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:121) ~[test-classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:143) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:477) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:486) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPBase.testRecoveryAndDoubleExecution(TestSCPBase.java:114) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPWithMeta.testRecoveryAndDoubleExecutionOnRsWithMeta(TestSCPWithMeta.java:36) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:102) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-18T18:25:25,575 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 1 exec state=pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, hasLock=true; ServerCrashProcedure 961a81da9827,36379,1731954257826, splitWal=true, meta=true 2024-11-18T18:25:25,575 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-18T18:25:25,575 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T18:25:25,575 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-18T18:25:25,575 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-18T18:25:25,575 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-18T18:25:25,576 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-18T18:25:25,576 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-18T18:25:25,576 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-18T18:25:25,577 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-18T18:25:25,580 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-18T18:25:25,580 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-18T18:25:25,581 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-18T18:25:25,582 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=false; ServerCrashProcedure 961a81da9827,36379,1731954257826, splitWal=true, meta=true 2024-11-18T18:25:25,582 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=false; ServerCrashProcedure 961a81da9827,36379,1731954257826, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-18T18:25:25,582 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-18T18:25:25,583 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33089 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T18:25:25,584 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33089 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T18:25:25,585 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=961a81da9827,36379,1731954257826, regionLocation=961a81da9827,36379,1731954257826, openSeqNum=2 2024-11-18T18:25:25,585 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=961a81da9827,36379,1731954257826, table=hbase:meta, region=1588230740 2024-11-18T18:25:25,585 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 961a81da9827,36379,1731954257826, state=OPEN 2024-11-18T18:25:25,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33089-0x10107179d9f0000, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T18:25:25,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38709-0x10107179d9f0001, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T18:25:25,588 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:25:25,588 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:25:25,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42895-0x10107179d9f0003, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T18:25:25,589 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:25:25,593 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731954355593 2024-11-18T18:25:25,593 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-18T18:25:25,594 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T18:25:25,594 DEBUG [Time-limited test {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1] 2024-11-18T18:25:25,595 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 961a81da9827,36379,1731954257826, splitWal=true, meta=true, isMeta: true 2024-11-18T18:25:25,595 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 961a81da9827:36379 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T18:25:25,596 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1, error=java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:25:25,596 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-18T18:25:25,596 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 from cache 2024-11-18T18:25:25,596 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 961a81da9827:36379 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:25:25,598 INFO [PEWorker-1 {}] master.SplitWALManager(105): 961a81da9827,36379,1731954257826 WAL count=1, meta=true 2024-11-18T18:25:25,598 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 961a81da9827%2C36379%2C1731954257826.meta.1731954259788.meta}] 2024-11-18T18:25:25,598 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-18T18:25:25,607 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=961a81da9827,42895,1731954257867 2024-11-18T18:25:25,610 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 961a81da9827%2C36379%2C1731954257826.meta.1731954259788.meta, worker=961a81da9827,42895,1731954257867}] 2024-11-18T18:25:25,702 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T18:25:25,702 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1] 2024-11-18T18:25:25,703 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 961a81da9827:36379 this server is in the failed servers list 2024-11-18T18:25:25,703 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=961a81da9827:36379 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:25:25,703 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:25:25,703 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 from cache 2024-11-18T18:25:25,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=59 2024-11-18T18:25:25,815 INFO [RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(299): Splitting hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/WALs/961a81da9827,36379,1731954257826-splitting/961a81da9827%2C36379%2C1731954257826.meta.1731954259788.meta, size=52.9 K (54214bytes) 2024-11-18T18:25:25,816 DEBUG [RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-18T18:25:25,816 DEBUG [RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-18T18:25:25,816 INFO [RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/WALs/961a81da9827,36379,1731954257826-splitting/961a81da9827%2C36379%2C1731954257826.meta.1731954259788.meta 2024-11-18T18:25:25,824 INFO [RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/WALs/961a81da9827,36379,1731954257826-splitting/961a81da9827%2C36379%2C1731954257826.meta.1731954259788.meta after 5ms 2024-11-18T18:25:25,837 INFO [RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(310): Open hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/WALs/961a81da9827,36379,1731954257826-splitting/961a81da9827%2C36379%2C1731954257826.meta.1731954259788.meta took 24ms 2024-11-18T18:25:25,859 DEBUG [RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 2024-11-18T18:25:25,867 INFO [RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-961a81da9827%2C36379%2C1731954257826.meta.1731954259788.meta.temp 2024-11-18T18:25:25,873 DEBUG [RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/WALs/961a81da9827,36379,1731954257826-splitting/961a81da9827%2C36379%2C1731954257826.meta.1731954259788.meta so closing down 2024-11-18T18:25:25,873 DEBUG [RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-18T18:25:25,874 INFO [RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.meta.1731954259788.meta.temp 2024-11-18T18:25:25,878 INFO [RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(145): 3 split writer threads finished 2024-11-18T18:25:25,912 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T18:25:25,912 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1] 2024-11-18T18:25:25,913 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 961a81da9827:36379 this server is in the failed servers list 2024-11-18T18:25:25,913 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=961a81da9827:36379 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:25:25,913 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:25:25,913 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 from cache 2024-11-18T18:25:25,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741864_1040 (size=53973) 2024-11-18T18:25:25,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741864_1040 (size=53973) 2024-11-18T18:25:25,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741864_1040 (size=53973) 2024-11-18T18:25:25,927 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.meta.1731954259788.meta.temp (wrote 57 edits, skipped 0 edits in 2 ms) 2024-11-18T18:25:25,934 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.meta.1731954259788.meta.temp to hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-18T18:25:25,941 INFO [RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(425): Processed 58 edits across 1 Regions in 88 ms; skipped=1; WAL=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/WALs/961a81da9827,36379,1731954257826-splitting/961a81da9827%2C36379%2C1731954257826.meta.1731954259788.meta, size=52.9 K, length=54214, corrupted=false, cancelled=false 2024-11-18T18:25:25,942 DEBUG [RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(428): Completed split of hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/WALs/961a81da9827,36379,1731954257826-splitting/961a81da9827%2C36379%2C1731954257826.meta.1731954259788.meta, journal: Splitting hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/WALs/961a81da9827,36379,1731954257826-splitting/961a81da9827%2C36379%2C1731954257826.meta.1731954259788.meta, size=52.9 K (54214bytes) at 1731954325815Finishing writing output for hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/WALs/961a81da9827,36379,1731954257826-splitting/961a81da9827%2C36379%2C1731954257826.meta.1731954259788.meta so closing down at 1731954325873 (+58 ms)Creating recovered edits writer path=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.meta.1731954259788.meta.temp at 1731954325874 (+1 ms)3 split writer threads finished at 1731954325878 (+4 ms)Closed recovered edits writer path=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.meta.1731954259788.meta.temp (wrote 57 edits, skipped 0 edits in 2 ms) at 1731954325927 (+49 ms)Rename recovered edits hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.meta.1731954259788.meta.temp to hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 at 1731954325934 (+7 ms)Processed 58 edits across 1 Regions in 88 ms; skipped=1; WAL=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/WALs/961a81da9827,36379,1731954257826-splitting/961a81da9827%2C36379%2C1731954257826.meta.1731954259788.meta, size=52.9 K, length=54214, corrupted=false, cancelled=false at 1731954325942 (+8 ms) 2024-11-18T18:25:25,942 DEBUG [RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/WALs/961a81da9827,36379,1731954257826-splitting/961a81da9827%2C36379%2C1731954257826.meta.1731954259788.meta 2024-11-18T18:25:25,944 DEBUG [RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-18T18:25:25,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33089 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-18T18:25:25,959 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/WALs/961a81da9827,36379,1731954257826-splitting/961a81da9827%2C36379%2C1731954257826.meta.1731954259788.meta to hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/oldWALs 2024-11-18T18:25:25,963 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-18T18:25:25,964 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 961a81da9827%2C36379%2C1731954257826.meta.1731954259788.meta, worker=961a81da9827,42895,1731954257867 in 349 msec 2024-11-18T18:25:25,966 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=961a81da9827,42895,1731954257867 2024-11-18T18:25:25,972 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-18T18:25:25,972 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 961a81da9827%2C36379%2C1731954257826.meta.1731954259788.meta, worker=961a81da9827,42895,1731954257867 in 368 msec 2024-11-18T18:25:25,976 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/WALs/961a81da9827,36379,1731954257826-splitting dir is empty, no logs to split. 2024-11-18T18:25:25,976 INFO [PEWorker-1 {}] master.SplitWALManager(105): 961a81da9827,36379,1731954257826 WAL count=0, meta=true 2024-11-18T18:25:25,976 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 961a81da9827,36379,1731954257826 WAL splitting is done? wals=0, meta=true 2024-11-18T18:25:25,976 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-18T18:25:25,976 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN_META, hasLock=true; ServerCrashProcedure 961a81da9827,36379,1731954257826, splitWal=true, meta=true 2024-11-18T18:25:25,976 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T18:25:25,976 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN_META, hasLock=true; ServerCrashProcedure 961a81da9827,36379,1731954257826, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T18:25:26,222 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T18:25:26,222 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1] 2024-11-18T18:25:26,223 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 961a81da9827:36379 this server is in the failed servers list 2024-11-18T18:25:26,223 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=961a81da9827:36379 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:25:26,223 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:25:26,223 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 from cache 2024-11-18T18:25:26,733 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T18:25:26,733 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1] 2024-11-18T18:25:26,733 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 961a81da9827:36379 this server is in the failed servers list 2024-11-18T18:25:26,734 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=961a81da9827:36379 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:25:26,734 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:25:26,734 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 from cache 2024-11-18T18:25:26,827 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=961a81da9827:33089 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 10 more 2024-11-18T18:25:27,742 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T18:25:27,742 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1] 2024-11-18T18:25:27,743 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 961a81da9827:36379 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T18:25:27,744 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 961a81da9827,36379,1731954257826 for '' in 1588230740 of hbase:meta failed, tries = 6, maxAttempts = 46, timeout = 60000 ms, time elapsed = 2150 ms java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-18T18:25:27,745 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1, error=java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:25:27,745 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-18T18:25:27,745 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 from cache 2024-11-18T18:25:27,745 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 961a81da9827:36379 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:25:29,763 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T18:25:29,763 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1] 2024-11-18T18:25:29,764 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 961a81da9827:36379 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T18:25:29,764 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 961a81da9827,36379,1731954257826 for '' in 1588230740 of hbase:meta failed, tries = 7, maxAttempts = 46, timeout = 60000 ms, time elapsed = 4170 ms java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-18T18:25:29,765 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1, error=java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:25:29,765 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-18T18:25:29,765 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 from cache 2024-11-18T18:25:29,765 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 961a81da9827:36379 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:25:33,783 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T18:25:33,783 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1] 2024-11-18T18:25:33,786 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 961a81da9827:36379 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T18:25:33,787 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 961a81da9827,36379,1731954257826 for '' in 1588230740 of hbase:meta failed, tries = 8, maxAttempts = 46, timeout = 60000 ms, time elapsed = 8192 ms java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-18T18:25:33,787 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1, error=java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:25:33,787 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-18T18:25:33,787 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 from cache 2024-11-18T18:25:33,788 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 961a81da9827:36379 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:25:43,803 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T18:25:43,803 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1] 2024-11-18T18:25:43,804 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 961a81da9827:36379 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T18:25:43,805 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 961a81da9827,36379,1731954257826 for '' in 1588230740 of hbase:meta failed, tries = 9, maxAttempts = 46, timeout = 60000 ms, time elapsed = 18210 ms java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-18T18:25:43,805 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1, error=java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:25:43,805 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-18T18:25:43,805 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 from cache 2024-11-18T18:25:43,805 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 961a81da9827:36379 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:25:46,051 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-18T18:25:53,892 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T18:25:53,892 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1] 2024-11-18T18:25:53,894 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 961a81da9827:36379 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T18:25:53,894 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 961a81da9827,36379,1731954257826 for '' in 1588230740 of hbase:meta failed, tries = 10, maxAttempts = 46, timeout = 60000 ms, time elapsed = 28300 ms java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-18T18:25:53,895 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1, error=java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:25:53,895 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-18T18:25:53,895 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 from cache 2024-11-18T18:25:53,895 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 961a81da9827:36379 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:26:03,983 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T18:26:03,983 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1] 2024-11-18T18:26:03,984 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 961a81da9827:36379 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T18:26:03,985 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 961a81da9827,36379,1731954257826 for '' in 1588230740 of hbase:meta failed, tries = 11, maxAttempts = 46, timeout = 60000 ms, time elapsed = 38390 ms java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-18T18:26:03,985 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1, error=java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:26:03,985 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-18T18:26:03,985 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 from cache 2024-11-18T18:26:03,985 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 961a81da9827:36379 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:26:14,063 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T18:26:14,063 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1] 2024-11-18T18:26:14,064 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 961a81da9827:36379 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T18:26:14,065 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 961a81da9827,36379,1731954257826 for '' in 1588230740 of hbase:meta failed, tries = 12, maxAttempts = 46, timeout = 60000 ms, time elapsed = 48471 ms java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-18T18:26:14,066 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1, error=java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:26:14,066 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-18T18:26:14,066 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 from cache 2024-11-18T18:26:14,066 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 961a81da9827:36379 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:26:16,052 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-18T18:26:19,124 WARN [regionserver/961a81da9827:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 22, running: 0 2024-11-18T18:26:19,739 INFO [regionserver/961a81da9827:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactedHFilesCleaner was stopped 2024-11-18T18:26:25,603 WARN [Time-limited test {}] procedure.MasterProcedureTestingUtility$3(130): Failed to load meta org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=13, exceptions: 2024-11-18T18:25:25.596Z, java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:25:25.703Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=961a81da9827:36379 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:25:25.914Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=961a81da9827:36379 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:25:26.223Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=961a81da9827:36379 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:25:26.734Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=961a81da9827:36379 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:25:27.745Z, java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:25:29.765Z, java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:25:33.787Z, java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:25:43.805Z, java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:25:53.895Z, java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:26:03.985Z, java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:26:14.066Z, java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:493) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:461) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:422) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.fullScanRegions(MetaTableAccessor.java:108) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionStateStore.visitMeta(RegionStateStore.java:108) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.loadMeta(AssignmentManager.java:1907) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.joinCluster(AssignmentManager.java:1769) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:126) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:121) ~[test-classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:143) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:477) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:486) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPBase.testRecoveryAndDoubleExecution(TestSCPBase.java:114) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPWithMeta.testRecoveryAndDoubleExecutionOnRsWithMeta(TestSCPWithMeta.java:36) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:102) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-18T18:26:25,605 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 2 exec state=pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN_META, hasLock=true; ServerCrashProcedure 961a81da9827,36379,1731954257826, splitWal=true, meta=true 2024-11-18T18:26:25,605 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-18T18:26:25,605 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T18:26:25,605 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-18T18:26:25,605 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-18T18:26:25,605 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-18T18:26:25,605 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-18T18:26:25,605 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-18T18:26:25,606 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-18T18:26:25,606 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-18T18:26:25,609 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-18T18:26:25,609 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-18T18:26:25,609 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-18T18:26:25,609 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-18T18:26:25,610 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-18T18:26:25,611 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, hasLock=false; ServerCrashProcedure 961a81da9827,36379,1731954257826, splitWal=true, meta=true 2024-11-18T18:26:25,611 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 961a81da9827%2C36379%2C1731954257826.meta.1731954259788.meta, worker=961a81da9827,42895,1731954257867 2024-11-18T18:26:25,611 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 961a81da9827%2C36379%2C1731954257826.meta.1731954259788.meta, worker=961a81da9827,42895,1731954257867 2024-11-18T18:26:25,611 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, hasLock=false; ServerCrashProcedure 961a81da9827,36379,1731954257826, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-18T18:26:25,611 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 961a81da9827%2C36379%2C1731954257826.meta.1731954259788.meta, worker=961a81da9827,42895,1731954257867 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T18:26:25,611 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 961a81da9827%2C36379%2C1731954257826.meta.1731954259788.meta, worker=961a81da9827,42895,1731954257867 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T18:26:25,611 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-18T18:26:25,612 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=961a81da9827,36379,1731954257826, regionLocation=961a81da9827,36379,1731954257826, openSeqNum=2 2024-11-18T18:26:25,612 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=961a81da9827,36379,1731954257826, table=hbase:meta, region=1588230740 2024-11-18T18:26:25,612 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 961a81da9827,36379,1731954257826, state=OPEN 2024-11-18T18:26:25,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33089-0x10107179d9f0000, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T18:26:25,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42895-0x10107179d9f0003, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T18:26:25,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38709-0x10107179d9f0001, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T18:26:25,614 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:26:25,614 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:26:25,614 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:26:25,615 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731954415615 2024-11-18T18:26:25,615 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-18T18:26:25,615 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T18:26:25,615 DEBUG [Time-limited test {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1] 2024-11-18T18:26:25,616 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 961a81da9827:36379 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T18:26:25,617 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1, error=java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:26:25,617 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-18T18:26:25,617 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 from cache 2024-11-18T18:26:25,617 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 961a81da9827:36379 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:26:25,617 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/WALs/961a81da9827,36379,1731954257826-splitting dir is empty, no logs to split. 2024-11-18T18:26:25,617 INFO [PEWorker-1 {}] master.SplitWALManager(105): 961a81da9827,36379,1731954257826 WAL count=0, meta=true 2024-11-18T18:26:25,618 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 961a81da9827,36379,1731954257826 WAL splitting is done? wals=0, meta=true 2024-11-18T18:26:25,618 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-18T18:26:25,620 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-18T18:26:25,620 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-18T18:26:25,620 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_GET_REGIONS, hasLock=true; ServerCrashProcedure 961a81da9827,36379,1731954257826, splitWal=true, meta=true 2024-11-18T18:26:25,620 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T18:26:25,620 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_GET_REGIONS, hasLock=true; ServerCrashProcedure 961a81da9827,36379,1731954257826, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T18:26:25,723 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T18:26:25,723 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1] 2024-11-18T18:26:25,723 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 961a81da9827:36379 this server is in the failed servers list 2024-11-18T18:26:25,724 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=961a81da9827:36379 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:26:25,724 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:26:25,724 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 from cache 2024-11-18T18:26:25,932 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T18:26:25,932 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1] 2024-11-18T18:26:25,933 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 961a81da9827:36379 this server is in the failed servers list 2024-11-18T18:26:25,933 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=961a81da9827:36379 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:26:25,933 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:26:25,933 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 from cache 2024-11-18T18:26:26,243 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T18:26:26,243 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1] 2024-11-18T18:26:26,243 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 961a81da9827:36379 this server is in the failed servers list 2024-11-18T18:26:26,244 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=961a81da9827:36379 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:26:26,244 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:26:26,244 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 from cache 2024-11-18T18:26:26,753 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T18:26:26,753 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1] 2024-11-18T18:26:26,753 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 961a81da9827:36379 this server is in the failed servers list 2024-11-18T18:26:26,754 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=961a81da9827:36379 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:26:26,754 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:26:26,754 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 from cache 2024-11-18T18:26:27,773 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T18:26:27,773 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1] 2024-11-18T18:26:27,774 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 961a81da9827:36379 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T18:26:27,774 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 961a81da9827,36379,1731954257826 for '' in 1588230740 of hbase:meta failed, tries = 6, maxAttempts = 46, timeout = 60000 ms, time elapsed = 2159 ms java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-18T18:26:27,775 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1, error=java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:26:27,775 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-18T18:26:27,775 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 from cache 2024-11-18T18:26:27,775 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 961a81da9827:36379 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:26:29,802 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T18:26:29,802 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1] 2024-11-18T18:26:29,803 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 961a81da9827:36379 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T18:26:29,804 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 961a81da9827,36379,1731954257826 for '' in 1588230740 of hbase:meta failed, tries = 7, maxAttempts = 46, timeout = 60000 ms, time elapsed = 4188 ms java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-18T18:26:29,804 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1, error=java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:26:29,804 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-18T18:26:29,804 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 from cache 2024-11-18T18:26:29,805 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 961a81da9827:36379 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:26:33,823 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T18:26:33,823 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1] 2024-11-18T18:26:33,824 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 961a81da9827:36379 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T18:26:33,824 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 961a81da9827,36379,1731954257826 for '' in 1588230740 of hbase:meta failed, tries = 8, maxAttempts = 46, timeout = 60000 ms, time elapsed = 8208 ms java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-18T18:26:33,825 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1, error=java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:26:33,825 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-18T18:26:33,825 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 from cache 2024-11-18T18:26:33,825 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 961a81da9827:36379 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:26:35,828 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=961a81da9827:33089 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 10 more 2024-11-18T18:26:43,852 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T18:26:43,853 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1] 2024-11-18T18:26:43,854 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 961a81da9827:36379 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T18:26:43,854 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 961a81da9827,36379,1731954257826 for '' in 1588230740 of hbase:meta failed, tries = 9, maxAttempts = 46, timeout = 60000 ms, time elapsed = 18238 ms java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-18T18:26:43,855 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1, error=java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:26:43,855 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-18T18:26:43,855 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 from cache 2024-11-18T18:26:43,859 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 961a81da9827:36379 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:26:46,053 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-18T18:26:53,893 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T18:26:53,893 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1] 2024-11-18T18:26:53,894 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 961a81da9827:36379 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T18:26:53,895 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 961a81da9827,36379,1731954257826 for '' in 1588230740 of hbase:meta failed, tries = 10, maxAttempts = 46, timeout = 60000 ms, time elapsed = 28279 ms java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-18T18:26:53,895 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1, error=java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:26:53,895 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-18T18:26:53,895 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 from cache 2024-11-18T18:26:53,895 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 961a81da9827:36379 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:27:03,912 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T18:27:03,912 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1] 2024-11-18T18:27:03,913 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 961a81da9827:36379 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T18:27:03,914 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 961a81da9827,36379,1731954257826 for '' in 1588230740 of hbase:meta failed, tries = 11, maxAttempts = 46, timeout = 60000 ms, time elapsed = 38298 ms java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-18T18:27:03,914 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1, error=java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:27:03,914 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-18T18:27:03,914 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 from cache 2024-11-18T18:27:03,915 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 961a81da9827:36379 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:27:13,973 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T18:27:13,973 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1] 2024-11-18T18:27:13,978 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 961a81da9827:36379 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T18:27:13,979 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 961a81da9827,36379,1731954257826 for '' in 1588230740 of hbase:meta failed, tries = 12, maxAttempts = 46, timeout = 60000 ms, time elapsed = 48363 ms java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-18T18:27:13,979 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1, error=java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:27:13,979 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-18T18:27:13,979 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 from cache 2024-11-18T18:27:13,979 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 961a81da9827:36379 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:27:16,054 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-18T18:27:17,342 WARN [RS:2;961a81da9827:42895 {}] monitoring.TaskMonitor(166): Status Processing ServerCrashProcedure of 961a81da9827,36379,1731954257826: status=Processing ServerCrashProcedure of 961a81da9827,36379,1731954257826 current State SERVER_CRASH_SPLIT_META_LOGS, state=RUNNING, startTime=1731954265543, completionTime=-1 appears to have been leaked 2024-11-18T18:27:17,342 WARN [RS:2;961a81da9827:42895 {}] monitoring.TaskMonitor(166): Status Processing ServerCrashProcedure of 961a81da9827,36379,1731954257826: status=Processing ServerCrashProcedure of 961a81da9827,36379,1731954257826 current State SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, state=RUNNING, startTime=1731954325582, completionTime=-1 appears to have been leaked 2024-11-18T18:27:25,623 WARN [Time-limited test {}] procedure.MasterProcedureTestingUtility$3(130): Failed to load meta org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=13, exceptions: 2024-11-18T18:26:25.617Z, java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:26:25.724Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=961a81da9827:36379 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:26:25.933Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=961a81da9827:36379 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:26:26.244Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=961a81da9827:36379 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:26:26.754Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=961a81da9827:36379 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:26:27.775Z, java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:26:29.804Z, java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:26:33.825Z, java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:26:43.855Z, java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:26:53.895Z, java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:27:03.914Z, java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:27:13.979Z, java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:493) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:461) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:422) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.fullScanRegions(MetaTableAccessor.java:108) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionStateStore.visitMeta(RegionStateStore.java:108) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.loadMeta(AssignmentManager.java:1907) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.joinCluster(AssignmentManager.java:1769) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:126) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:121) ~[test-classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:143) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:477) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:486) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPBase.testRecoveryAndDoubleExecution(TestSCPBase.java:114) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPWithMeta.testRecoveryAndDoubleExecutionOnRsWithMeta(TestSCPWithMeta.java:36) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:102) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-18T18:27:25,624 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 3 exec state=pid=57, state=WAITING:SERVER_CRASH_GET_REGIONS, hasLock=true; ServerCrashProcedure 961a81da9827,36379,1731954257826, splitWal=true, meta=true 2024-11-18T18:27:25,624 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-18T18:27:25,624 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T18:27:25,624 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-18T18:27:25,624 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-18T18:27:25,625 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-18T18:27:25,625 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-18T18:27:25,625 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-18T18:27:25,625 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-18T18:27:25,626 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-18T18:27:25,627 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-18T18:27:25,627 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-18T18:27:25,627 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-18T18:27:25,628 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-18T18:27:25,628 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-18T18:27:25,629 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN_META, hasLock=false; ServerCrashProcedure 961a81da9827,36379,1731954257826, splitWal=true, meta=true 2024-11-18T18:27:25,629 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 961a81da9827%2C36379%2C1731954257826.meta.1731954259788.meta, worker=961a81da9827,42895,1731954257867 2024-11-18T18:27:25,629 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 961a81da9827%2C36379%2C1731954257826.meta.1731954259788.meta, worker=961a81da9827,42895,1731954257867 2024-11-18T18:27:25,629 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN_META, hasLock=false; ServerCrashProcedure 961a81da9827,36379,1731954257826, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-18T18:27:25,629 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 961a81da9827%2C36379%2C1731954257826.meta.1731954259788.meta, worker=961a81da9827,42895,1731954257867 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T18:27:25,629 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 961a81da9827%2C36379%2C1731954257826.meta.1731954259788.meta, worker=961a81da9827,42895,1731954257867 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T18:27:25,629 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-18T18:27:25,630 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=961a81da9827,36379,1731954257826, regionLocation=961a81da9827,36379,1731954257826, openSeqNum=2 2024-11-18T18:27:25,630 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=961a81da9827,36379,1731954257826, table=hbase:meta, region=1588230740 2024-11-18T18:27:25,630 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 961a81da9827,36379,1731954257826, state=OPEN 2024-11-18T18:27:25,632 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33089-0x10107179d9f0000, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T18:27:25,632 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38709-0x10107179d9f0001, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T18:27:25,632 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42895-0x10107179d9f0003, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T18:27:25,632 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:27:25,632 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:27:25,632 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:27:25,633 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731954475633 2024-11-18T18:27:25,633 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-18T18:27:25,633 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-18T18:27:25,634 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-18T18:27:25,634 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T18:27:25,634 DEBUG [Time-limited test {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1] 2024-11-18T18:27:25,635 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 961a81da9827:36379 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T18:27:25,636 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1, error=java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:27:25,636 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-18T18:27:25,636 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 from cache 2024-11-18T18:27:25,636 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 961a81da9827:36379 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:27:25,637 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T18:27:25,638 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-18T18:27:25,743 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T18:27:25,743 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1] 2024-11-18T18:27:25,743 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 961a81da9827:36379 this server is in the failed servers list 2024-11-18T18:27:25,743 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=961a81da9827:36379 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:27:25,743 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:27:25,743 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 from cache 2024-11-18T18:27:25,788 DEBUG [961a81da9827:33089 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-18T18:27:25,789 DEBUG [961a81da9827:33089 {}] balancer.BalancerClusterState(204): Hosts are {961a81da9827=0} racks are {/default-rack=0} 2024-11-18T18:27:25,789 DEBUG [961a81da9827:33089 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-18T18:27:25,789 DEBUG [961a81da9827:33089 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-18T18:27:25,789 DEBUG [961a81da9827:33089 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-18T18:27:25,789 DEBUG [961a81da9827:33089 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-18T18:27:25,789 INFO [961a81da9827:33089 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-18T18:27:25,789 INFO [961a81da9827:33089 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-18T18:27:25,789 DEBUG [961a81da9827:33089 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-18T18:27:25,790 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=961a81da9827,42895,1731954257867 2024-11-18T18:27:25,791 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 961a81da9827,42895,1731954257867, state=OPENING 2024-11-18T18:27:25,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33089-0x10107179d9f0000, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T18:27:25,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42895-0x10107179d9f0003, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T18:27:25,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38709-0x10107179d9f0001, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T18:27:25,793 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:27:25,793 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:27:25,793 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T18:27:25,793 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:27:25,793 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=961a81da9827,42895,1731954257867}] 2024-11-18T18:27:25,949 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52475, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-18T18:27:25,953 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T18:27:25,953 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=961a81da9827,42895,1731954257867, seqNum=-1] 2024-11-18T18:27:25,954 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T18:27:25,955 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32833, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T18:27:25,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Scan size: 121 connection: 172.17.0.2:32833 deadline: 1731954505635, exception=org.apache.hadoop.hbase.NotServingRegionException: hbase:meta,,1 is not online on 961a81da9827,42895,1731954257867 2024-11-18T18:27:25,962 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,42895,1731954257867, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=961a81da9827,42895,1731954257867, seqNum=-1, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: hbase:meta,,1 is not online on 961a81da9827,42895,1731954257867 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T18:27:25,962 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,42895,1731954257867, seqNum=-1 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: hbase:meta,,1 is not online on 961a81da9827,42895,1731954257867 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T18:27:25,962 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=961a81da9827,42895,1731954257867, seqNum=-1 from cache 2024-11-18T18:27:25,962 INFO [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-18T18:27:25,962 INFO [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T18:27:25,962 INFO [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=61}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-18T18:27:25,965 INFO [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=961a81da9827%2C42895%2C1731954257867.meta, suffix=.meta, logDir=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/WALs/961a81da9827,42895,1731954257867, archiveDir=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/oldWALs, maxLogs=32 2024-11-18T18:27:25,987 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=61}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/WALs/961a81da9827,42895,1731954257867/961a81da9827%2C42895%2C1731954257867.meta.1731954445967.meta, exclude list is [], retry=0 2024-11-18T18:27:25,991 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34047,DS-1029886b-6b51-4ae4-81a3-156afc7de901,DISK] 2024-11-18T18:27:25,992 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44191,DS-39ff43bf-c18b-4b86-b37c-eca99a433383,DISK] 2024-11-18T18:27:25,992 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41713,DS-c75b1747-362b-431f-b47d-543a4f882cdb,DISK] 2024-11-18T18:27:26,002 INFO [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/WALs/961a81da9827,42895,1731954257867/961a81da9827%2C42895%2C1731954257867.meta.1731954445967.meta 2024-11-18T18:27:26,002 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41643:41643),(127.0.0.1/127.0.0.1:38097:38097),(127.0.0.1/127.0.0.1:34389:34389)] 2024-11-18T18:27:26,003 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-18T18:27:26,003 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=61}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-18T18:27:26,003 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-18T18:27:26,004 INFO [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-18T18:27:26,004 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-18T18:27:26,004 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:27:26,004 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-18T18:27:26,004 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-18T18:27:26,009 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-18T18:27:26,011 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-18T18:27:26,011 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:27:26,012 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T18:27:26,012 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-18T18:27:26,013 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-18T18:27:26,013 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:27:26,014 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T18:27:26,014 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-18T18:27:26,015 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-18T18:27:26,015 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:27:26,016 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T18:27:26,016 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-18T18:27:26,017 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-18T18:27:26,017 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:27:26,018 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T18:27:26,018 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-18T18:27:26,019 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/hbase/meta/1588230740 2024-11-18T18:27:26,022 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/hbase/meta/1588230740 2024-11-18T18:27:26,023 INFO [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-18T18:27:26,042 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5793): Applied 264, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=60, path=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-18T18:27:26,046 INFO [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=49.58 KB heapSize=69.98 KB 2024-11-18T18:27:26,159 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/hbase/meta/1588230740/.tmp/info/02e4d6e3ec5a4f2090a7b6987cb8a637 is 268, key is testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731954260238.22971deb0db38b7cccde5c3bc0868f22./info:regioninfo/1731954261598/Put/seqid=0 2024-11-18T18:27:26,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741866_1042 (size=58053) 2024-11-18T18:27:26,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741866_1042 (size=58053) 2024-11-18T18:27:26,214 INFO [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.31 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/hbase/meta/1588230740/.tmp/info/02e4d6e3ec5a4f2090a7b6987cb8a637 2024-11-18T18:27:26,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741866_1042 (size=58053) 2024-11-18T18:27:26,272 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T18:27:26,272 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=961a81da9827,42895,1731954257867, seqNum=-1] 2024-11-18T18:27:26,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Scan size: 121 connection: 172.17.0.2:32833 deadline: 1731954505634, exception=org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on 961a81da9827,42895,1731954257867 2024-11-18T18:27:26,274 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,42895,1731954257867, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=961a81da9827,42895,1731954257867, seqNum=-1, error=org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on 961a81da9827,42895,1731954257867 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T18:27:26,275 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,42895,1731954257867, seqNum=-1 is org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on 961a81da9827,42895,1731954257867 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T18:27:26,275 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=961a81da9827,42895,1731954257867, seqNum=-1 from cache 2024-11-18T18:27:26,299 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/hbase/meta/1588230740/.tmp/ns/b63beffb3acc415e9ffb52a3bd7c0fda is 43, key is default/ns:d/1731954259962/Put/seqid=0 2024-11-18T18:27:26,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741867_1043 (size=5153) 2024-11-18T18:27:26,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741867_1043 (size=5153) 2024-11-18T18:27:26,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741867_1043 (size=5153) 2024-11-18T18:27:26,315 INFO [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/hbase/meta/1588230740/.tmp/ns/b63beffb3acc415e9ffb52a3bd7c0fda 2024-11-18T18:27:26,352 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/hbase/meta/1588230740/.tmp/table/e3e567f5799a4e0682ff948d3a29cb93 is 105, key is testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/table:state/1731954262163/Put/seqid=0 2024-11-18T18:27:26,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741868_1044 (size=5620) 2024-11-18T18:27:26,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741868_1044 (size=5620) 2024-11-18T18:27:26,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741868_1044 (size=5620) 2024-11-18T18:27:26,375 INFO [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=202 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/hbase/meta/1588230740/.tmp/table/e3e567f5799a4e0682ff948d3a29cb93 2024-11-18T18:27:26,387 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/hbase/meta/1588230740/.tmp/info/02e4d6e3ec5a4f2090a7b6987cb8a637 as hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/hbase/meta/1588230740/info/02e4d6e3ec5a4f2090a7b6987cb8a637 2024-11-18T18:27:26,398 INFO [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/hbase/meta/1588230740/info/02e4d6e3ec5a4f2090a7b6987cb8a637, entries=260, sequenceid=60, filesize=56.7 K 2024-11-18T18:27:26,402 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/hbase/meta/1588230740/.tmp/ns/b63beffb3acc415e9ffb52a3bd7c0fda as hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/hbase/meta/1588230740/ns/b63beffb3acc415e9ffb52a3bd7c0fda 2024-11-18T18:27:26,422 INFO [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/hbase/meta/1588230740/ns/b63beffb3acc415e9ffb52a3bd7c0fda, entries=2, sequenceid=60, filesize=5.0 K 2024-11-18T18:27:26,424 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/hbase/meta/1588230740/.tmp/table/e3e567f5799a4e0682ff948d3a29cb93 as hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/hbase/meta/1588230740/table/e3e567f5799a4e0682ff948d3a29cb93 2024-11-18T18:27:26,433 INFO [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/hbase/meta/1588230740/table/e3e567f5799a4e0682ff948d3a29cb93, entries=2, sequenceid=60, filesize=5.5 K 2024-11-18T18:27:26,439 INFO [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~49.58 KB/50772, heapSize ~69.69 KB/71360, currentSize=0 B/0 for 1588230740 in 392ms, sequenceid=60, compaction requested=false; wal=null 2024-11-18T18:27:26,442 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-18T18:27:26,444 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-18T18:27:26,444 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-18T18:27:26,445 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T18:27:26,447 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-18T18:27:26,451 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/hbase/meta/1588230740/recovered.edits/60.seqid, newMaxSeqId=60, maxSeqId=1 2024-11-18T18:27:26,453 INFO [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=61; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59564347, jitterRate=-0.11242206394672394}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T18:27:26,453 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-18T18:27:26,454 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731954446004Writing region info on filesystem at 1731954446004Initializing all the Stores at 1731954446008 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731954446008Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731954446009 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954446009Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731954446009Obtaining lock to block concurrent updates at 1731954446046 (+37 ms)Preparing flush snapshotting stores in 1588230740 at 1731954446047 (+1 ms)Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=50772, getHeapSize=71600, getOffHeapSize=0, getCellsCount=264 at 1731954446050 (+3 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731954446051 (+1 ms)Flushing 1588230740/info: creating writer at 1731954446054 (+3 ms)Flushing 1588230740/info: appending metadata at 1731954446148 (+94 ms)Flushing 1588230740/info: closing flushed file at 1731954446152 (+4 ms)Flushing 1588230740/ns: creating writer at 1731954446279 (+127 ms)Flushing 1588230740/ns: appending metadata at 1731954446297 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1731954446297Flushing 1588230740/table: creating writer at 1731954446327 (+30 ms)Flushing 1588230740/table: appending metadata at 1731954446350 (+23 ms)Flushing 1588230740/table: closing flushed file at 1731954446351 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@656ab87e: reopening flushed file at 1731954446385 (+34 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@750eb976: reopening flushed file at 1731954446400 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d4308cd: reopening flushed file at 1731954446423 (+23 ms)Finished flush of dataSize ~49.58 KB/50772, heapSize ~69.69 KB/71360, currentSize=0 B/0 for 1588230740 in 392ms, sequenceid=60, compaction requested=false; wal=null at 1731954446439 (+16 ms)Cleaning up temporary data from old regions at 1731954446444 (+5 ms)Running coprocessor post-open hooks at 1731954446453 (+9 ms)Region opened successfully at 1731954446454 (+1 ms) 2024-11-18T18:27:26,456 INFO [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=61, masterSystemTime=1731954445946 2024-11-18T18:27:26,460 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-18T18:27:26,460 INFO [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-18T18:27:26,461 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=61, regionLocation=961a81da9827,42895,1731954257867 2024-11-18T18:27:26,462 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 961a81da9827,42895,1731954257867, state=OPEN 2024-11-18T18:27:26,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33089-0x10107179d9f0000, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T18:27:26,464 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=61, ppid=60, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=961a81da9827,42895,1731954257867 2024-11-18T18:27:26,465 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:27:26,465 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42895-0x10107179d9f0003, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T18:27:26,465 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38709-0x10107179d9f0001, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T18:27:26,465 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:27:26,465 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:27:26,469 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-18T18:27:26,469 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=961a81da9827,42895,1731954257867 in 672 msec 2024-11-18T18:27:26,473 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=57 2024-11-18T18:27:26,473 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 836 msec 2024-11-18T18:27:26,783 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T18:27:26,783 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=961a81da9827,42895,1731954257867, seqNum=-1] 2024-11-18T18:27:26,907 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8bb7ae0747eda2290799ca8fa83f9ede, regionState=OPEN, lastHost=961a81da9827,38709,1731954257723, regionLocation=961a81da9827,38709,1731954257723, openSeqNum=2 2024-11-18T18:27:26,907 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22971deb0db38b7cccde5c3bc0868f22, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=2 2024-11-18T18:27:26,907 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43138a6a12c2efea7093064622bb3cd8, regionState=OPEN, lastHost=961a81da9827,36379,1731954257826, regionLocation=961a81da9827,36379,1731954257826, openSeqNum=2 2024-11-18T18:27:26,907 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb099f0975ebdb618ef8e3b2c263bbca, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=2 2024-11-18T18:27:26,908 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bcf8695060a326b7d24615ba6cb932b1, regionState=OPEN, lastHost=961a81da9827,36379,1731954257826, regionLocation=961a81da9827,36379,1731954257826, openSeqNum=2 2024-11-18T18:27:26,908 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb91ff28ef3c18f0dc0d3f26aac68eca, regionState=OPEN, lastHost=961a81da9827,38709,1731954257723, regionLocation=961a81da9827,38709,1731954257723, openSeqNum=2 2024-11-18T18:27:26,908 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8b4dd2a6d8dd59085c547b55aac0de6, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=2 2024-11-18T18:27:26,909 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40a213dc6b2a0fd809c71612c437ab83, regionState=OPEN, lastHost=961a81da9827,36379,1731954257826, regionLocation=961a81da9827,36379,1731954257826, openSeqNum=2 2024-11-18T18:27:26,909 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f39571a93e25091090ea0831d54936b5, regionState=OPEN, lastHost=961a81da9827,38709,1731954257723, regionLocation=961a81da9827,38709,1731954257723, openSeqNum=2 2024-11-18T18:27:26,909 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97a938e093c14701871e85a1fbb6e07d, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=2 2024-11-18T18:27:26,910 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c3e2f3fb6516fcaaef4d0a2c8b63842d, regionState=OPEN, lastHost=961a81da9827,36379,1731954257826, regionLocation=961a81da9827,36379,1731954257826, openSeqNum=2 2024-11-18T18:27:26,910 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97377a91d35f8b242120f603c456c41e, regionState=OPEN, lastHost=961a81da9827,38709,1731954257723, regionLocation=961a81da9827,38709,1731954257723, openSeqNum=2 2024-11-18T18:27:26,910 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a244d0ab318eb4dd77bc5235828d9d51, regionState=OPEN, lastHost=961a81da9827,36379,1731954257826, regionLocation=961a81da9827,36379,1731954257826, openSeqNum=2 2024-11-18T18:27:26,910 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=012c277c99c564dd90e8e5f6850b6ce0, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=2 2024-11-18T18:27:26,910 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=140f5077d3b730b643b558ebe8a13abd, regionState=OPEN, lastHost=961a81da9827,38709,1731954257723, regionLocation=961a81da9827,38709,1731954257723, openSeqNum=2 2024-11-18T18:27:26,911 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e69ef2147d87d1b83dbcbb4705cbabe, regionState=OPEN, lastHost=961a81da9827,36379,1731954257826, regionLocation=961a81da9827,36379,1731954257826, openSeqNum=2 2024-11-18T18:27:26,911 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb8fa4293ea5e58c18dbc5b978355ca9, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=2 2024-11-18T18:27:26,911 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69ae79c6f9eeed559fa88701f773706c, regionState=OPEN, lastHost=961a81da9827,38709,1731954257723, regionLocation=961a81da9827,38709,1731954257723, openSeqNum=2 2024-11-18T18:27:26,911 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=75a13ed51395b1016b90c6b0d90a7d0a, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=2 2024-11-18T18:27:26,911 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aaf2435418703b2a31c1190af1560ba8, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=2 2024-11-18T18:27:26,911 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1ad74f79ba4d7fd1619c83a55106331, regionState=OPEN, lastHost=961a81da9827,36379,1731954257826, regionLocation=961a81da9827,36379,1731954257826, openSeqNum=2 2024-11-18T18:27:26,912 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=81f0db1bea1e11879dc1cafb06c0756d, regionState=OPEN, lastHost=961a81da9827,36379,1731954257826, regionLocation=961a81da9827,36379,1731954257826, openSeqNum=2 2024-11-18T18:27:26,912 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2a8fcaf8905dcde0537806fb16b9ff4c, regionState=OPEN, lastHost=961a81da9827,38709,1731954257723, regionLocation=961a81da9827,38709,1731954257723, openSeqNum=2 2024-11-18T18:27:26,912 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=579ee42a94597fdc1db9a4820a53ee64, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=2 2024-11-18T18:27:26,912 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=779798cd60db955ed78a4329cac08969, regionState=OPEN, lastHost=961a81da9827,36379,1731954257826, regionLocation=961a81da9827,36379,1731954257826, openSeqNum=2 2024-11-18T18:27:26,912 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=08ccc32db22fb75f5466ad774ad9e8ba, regionState=OPEN, lastHost=961a81da9827,38709,1731954257723, regionLocation=961a81da9827,38709,1731954257723, openSeqNum=2 2024-11-18T18:27:26,912 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-18T18:27:26,912 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731954506912 2024-11-18T18:27:26,912 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731954566912 2024-11-18T18:27:26,912 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 1.2790 sec 2024-11-18T18:27:26,913 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 961a81da9827,36379,1731954257826 had 10 regions 2024-11-18T18:27:26,913 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-18T18:27:26,913 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 961a81da9827,36379,1731954257826, splitWal=true, meta=true 2024-11-18T18:27:26,913 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T18:27:26,914 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 961a81da9827,36379,1731954257826, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T18:27:27,163 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 4 exec state=pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 961a81da9827,36379,1731954257826, splitWal=true, meta=true 2024-11-18T18:27:27,163 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-18T18:27:27,163 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T18:27:27,163 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-18T18:27:27,163 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-18T18:27:27,164 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-18T18:27:27,164 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-18T18:27:27,164 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-18T18:27:27,164 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-18T18:27:27,168 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 3 msec 2024-11-18T18:27:27,182 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-18T18:27:27,182 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-18T18:27:27,182 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-18T18:27:27,182 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T18:27:27,182 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T18:27:27,183 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-18T18:27:27,184 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-18T18:27:27,185 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_GET_REGIONS, hasLock=false; ServerCrashProcedure 961a81da9827,36379,1731954257826, splitWal=true, meta=true 2024-11-18T18:27:27,185 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 961a81da9827%2C36379%2C1731954257826.meta.1731954259788.meta, worker=961a81da9827,42895,1731954257867 2024-11-18T18:27:27,185 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 961a81da9827%2C36379%2C1731954257826.meta.1731954259788.meta, worker=961a81da9827,42895,1731954257867 2024-11-18T18:27:27,189 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33089 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T18:27:27,189 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T18:27:27,189 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=961a81da9827,42895,1731954257867 2024-11-18T18:27:27,189 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_GET_REGIONS, hasLock=false; ServerCrashProcedure 961a81da9827,36379,1731954257826, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-18T18:27:27,189 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 961a81da9827%2C36379%2C1731954257826.meta.1731954259788.meta, worker=961a81da9827,42895,1731954257867 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T18:27:27,189 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 961a81da9827%2C36379%2C1731954257826.meta.1731954259788.meta, worker=961a81da9827,42895,1731954257867 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T18:27:27,189 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T18:27:27,189 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=961a81da9827,42895,1731954257867 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T18:27:27,189 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33089 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T18:27:27,190 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 21 msec 2024-11-18T18:27:27,218 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=61 2024-11-18T18:27:27,218 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=961a81da9827,42895,1731954257867, table=hbase:meta, region=1588230740 2024-11-18T18:27:27,218 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 961a81da9827,42895,1731954257867, state=OPEN 2024-11-18T18:27:27,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33089-0x10107179d9f0000, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T18:27:27,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38709-0x10107179d9f0001, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T18:27:27,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42895-0x10107179d9f0003, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T18:27:27,221 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:27:27,221 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:27:27,221 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:27:27,235 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731954477235 2024-11-18T18:27:27,235 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-18T18:27:27,241 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8bb7ae0747eda2290799ca8fa83f9ede, regionState=OPEN, lastHost=961a81da9827,38709,1731954257723, regionLocation=961a81da9827,38709,1731954257723, openSeqNum=2 2024-11-18T18:27:27,241 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22971deb0db38b7cccde5c3bc0868f22, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=2 2024-11-18T18:27:27,241 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43138a6a12c2efea7093064622bb3cd8, regionState=OPEN, lastHost=961a81da9827,36379,1731954257826, regionLocation=961a81da9827,36379,1731954257826, openSeqNum=2 2024-11-18T18:27:27,245 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb099f0975ebdb618ef8e3b2c263bbca, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=2 2024-11-18T18:27:27,246 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bcf8695060a326b7d24615ba6cb932b1, regionState=OPEN, lastHost=961a81da9827,36379,1731954257826, regionLocation=961a81da9827,36379,1731954257826, openSeqNum=2 2024-11-18T18:27:27,246 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb91ff28ef3c18f0dc0d3f26aac68eca, regionState=OPEN, lastHost=961a81da9827,38709,1731954257723, regionLocation=961a81da9827,38709,1731954257723, openSeqNum=2 2024-11-18T18:27:27,246 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8b4dd2a6d8dd59085c547b55aac0de6, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=2 2024-11-18T18:27:27,246 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40a213dc6b2a0fd809c71612c437ab83, regionState=OPEN, lastHost=961a81da9827,36379,1731954257826, regionLocation=961a81da9827,36379,1731954257826, openSeqNum=2 2024-11-18T18:27:27,246 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f39571a93e25091090ea0831d54936b5, regionState=OPEN, lastHost=961a81da9827,38709,1731954257723, regionLocation=961a81da9827,38709,1731954257723, openSeqNum=2 2024-11-18T18:27:27,246 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97a938e093c14701871e85a1fbb6e07d, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=2 2024-11-18T18:27:27,246 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c3e2f3fb6516fcaaef4d0a2c8b63842d, regionState=OPEN, lastHost=961a81da9827,36379,1731954257826, regionLocation=961a81da9827,36379,1731954257826, openSeqNum=2 2024-11-18T18:27:27,247 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97377a91d35f8b242120f603c456c41e, regionState=OPEN, lastHost=961a81da9827,38709,1731954257723, regionLocation=961a81da9827,38709,1731954257723, openSeqNum=2 2024-11-18T18:27:27,247 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a244d0ab318eb4dd77bc5235828d9d51, regionState=OPEN, lastHost=961a81da9827,36379,1731954257826, regionLocation=961a81da9827,36379,1731954257826, openSeqNum=2 2024-11-18T18:27:27,247 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=012c277c99c564dd90e8e5f6850b6ce0, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=2 2024-11-18T18:27:27,247 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=140f5077d3b730b643b558ebe8a13abd, regionState=OPEN, lastHost=961a81da9827,38709,1731954257723, regionLocation=961a81da9827,38709,1731954257723, openSeqNum=2 2024-11-18T18:27:27,247 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e69ef2147d87d1b83dbcbb4705cbabe, regionState=OPEN, lastHost=961a81da9827,36379,1731954257826, regionLocation=961a81da9827,36379,1731954257826, openSeqNum=2 2024-11-18T18:27:27,247 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb8fa4293ea5e58c18dbc5b978355ca9, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=2 2024-11-18T18:27:27,247 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69ae79c6f9eeed559fa88701f773706c, regionState=OPEN, lastHost=961a81da9827,38709,1731954257723, regionLocation=961a81da9827,38709,1731954257723, openSeqNum=2 2024-11-18T18:27:27,247 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=75a13ed51395b1016b90c6b0d90a7d0a, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=2 2024-11-18T18:27:27,247 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aaf2435418703b2a31c1190af1560ba8, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=2 2024-11-18T18:27:27,247 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1ad74f79ba4d7fd1619c83a55106331, regionState=OPEN, lastHost=961a81da9827,36379,1731954257826, regionLocation=961a81da9827,36379,1731954257826, openSeqNum=2 2024-11-18T18:27:27,247 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=81f0db1bea1e11879dc1cafb06c0756d, regionState=OPEN, lastHost=961a81da9827,36379,1731954257826, regionLocation=961a81da9827,36379,1731954257826, openSeqNum=2 2024-11-18T18:27:27,248 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2a8fcaf8905dcde0537806fb16b9ff4c, regionState=OPEN, lastHost=961a81da9827,38709,1731954257723, regionLocation=961a81da9827,38709,1731954257723, openSeqNum=2 2024-11-18T18:27:27,248 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=579ee42a94597fdc1db9a4820a53ee64, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=2 2024-11-18T18:27:27,248 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=779798cd60db955ed78a4329cac08969, regionState=OPEN, lastHost=961a81da9827,36379,1731954257826, regionLocation=961a81da9827,36379,1731954257826, openSeqNum=2 2024-11-18T18:27:27,248 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=08ccc32db22fb75f5466ad774ad9e8ba, regionState=OPEN, lastHost=961a81da9827,38709,1731954257723, regionLocation=961a81da9827,38709,1731954257723, openSeqNum=2 2024-11-18T18:27:27,248 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-18T18:27:27,248 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731954507248 2024-11-18T18:27:27,248 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731954567248 2024-11-18T18:27:27,248 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 13 msec 2024-11-18T18:27:27,249 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 961a81da9827,36379,1731954257826 had 9 regions 2024-11-18T18:27:27,249 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-18T18:27:27,250 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 961a81da9827,36379,1731954257826, splitWal=true, meta=true, isMeta: false 2024-11-18T18:27:27,253 INFO [PEWorker-1 {}] master.SplitWALManager(105): 961a81da9827,36379,1731954257826 WAL count=1, meta=false 2024-11-18T18:27:27,253 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 961a81da9827%2C36379%2C1731954257826.1731954259295}] 2024-11-18T18:27:27,253 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-18T18:27:27,253 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_WALS_DIR, hasLock=true; ServerCrashProcedure 961a81da9827,36379,1731954257826, splitWal=true, meta=true 2024-11-18T18:27:27,253 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T18:27:27,254 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_WALS_DIR, hasLock=true; ServerCrashProcedure 961a81da9827,36379,1731954257826, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T18:27:27,499 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 5 exec state=pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_WALS_DIR, hasLock=true; ServerCrashProcedure 961a81da9827,36379,1731954257826, splitWal=true, meta=true 2024-11-18T18:27:27,499 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-18T18:27:27,499 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T18:27:27,499 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-18T18:27:27,499 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-18T18:27:27,500 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-18T18:27:27,500 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-18T18:27:27,500 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-18T18:27:27,501 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-18T18:27:27,503 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33089 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T18:27:27,504 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33089 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T18:27:27,509 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 8 msec 2024-11-18T18:27:27,511 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-18T18:27:27,512 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-18T18:27:27,512 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-18T18:27:27,512 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T18:27:27,512 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T18:27:27,512 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-18T18:27:27,513 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-18T18:27:27,514 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=false; ServerCrashProcedure 961a81da9827,36379,1731954257826, splitWal=true, meta=true 2024-11-18T18:27:27,514 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 961a81da9827%2C36379%2C1731954257826.meta.1731954259788.meta, worker=961a81da9827,42895,1731954257867 2024-11-18T18:27:27,514 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 961a81da9827%2C36379%2C1731954257826.meta.1731954259788.meta, worker=961a81da9827,42895,1731954257867 2024-11-18T18:27:27,514 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T18:27:27,514 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=961a81da9827,42895,1731954257867 2024-11-18T18:27:27,514 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=false; ServerCrashProcedure 961a81da9827,36379,1731954257826, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-18T18:27:27,514 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 961a81da9827%2C36379%2C1731954257826.meta.1731954259788.meta, worker=961a81da9827,42895,1731954257867 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T18:27:27,514 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 961a81da9827%2C36379%2C1731954257826.meta.1731954259788.meta, worker=961a81da9827,42895,1731954257867 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T18:27:27,514 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T18:27:27,514 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=961a81da9827,42895,1731954257867 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T18:27:27,515 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-18T18:27:27,559 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=61 2024-11-18T18:27:27,559 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=961a81da9827,42895,1731954257867, table=hbase:meta, region=1588230740 2024-11-18T18:27:27,559 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 961a81da9827,42895,1731954257867, state=OPEN 2024-11-18T18:27:27,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38709-0x10107179d9f0001, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T18:27:27,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42895-0x10107179d9f0003, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T18:27:27,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33089-0x10107179d9f0000, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T18:27:27,562 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:27:27,562 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:27:27,562 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:27:27,699 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731954477699 2024-11-18T18:27:27,700 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-18T18:27:27,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8bb7ae0747eda2290799ca8fa83f9ede, regionState=OPEN, lastHost=961a81da9827,38709,1731954257723, regionLocation=961a81da9827,38709,1731954257723, openSeqNum=2 2024-11-18T18:27:27,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22971deb0db38b7cccde5c3bc0868f22, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=2 2024-11-18T18:27:27,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43138a6a12c2efea7093064622bb3cd8, regionState=OPEN, lastHost=961a81da9827,36379,1731954257826, regionLocation=961a81da9827,36379,1731954257826, openSeqNum=2 2024-11-18T18:27:27,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb099f0975ebdb618ef8e3b2c263bbca, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=2 2024-11-18T18:27:27,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bcf8695060a326b7d24615ba6cb932b1, regionState=OPEN, lastHost=961a81da9827,36379,1731954257826, regionLocation=961a81da9827,36379,1731954257826, openSeqNum=2 2024-11-18T18:27:27,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb91ff28ef3c18f0dc0d3f26aac68eca, regionState=OPEN, lastHost=961a81da9827,38709,1731954257723, regionLocation=961a81da9827,38709,1731954257723, openSeqNum=2 2024-11-18T18:27:27,724 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8b4dd2a6d8dd59085c547b55aac0de6, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=2 2024-11-18T18:27:27,724 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40a213dc6b2a0fd809c71612c437ab83, regionState=OPEN, lastHost=961a81da9827,36379,1731954257826, regionLocation=961a81da9827,36379,1731954257826, openSeqNum=2 2024-11-18T18:27:27,724 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f39571a93e25091090ea0831d54936b5, regionState=OPEN, lastHost=961a81da9827,38709,1731954257723, regionLocation=961a81da9827,38709,1731954257723, openSeqNum=2 2024-11-18T18:27:27,724 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97a938e093c14701871e85a1fbb6e07d, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=2 2024-11-18T18:27:27,724 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c3e2f3fb6516fcaaef4d0a2c8b63842d, regionState=OPEN, lastHost=961a81da9827,36379,1731954257826, regionLocation=961a81da9827,36379,1731954257826, openSeqNum=2 2024-11-18T18:27:27,724 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97377a91d35f8b242120f603c456c41e, regionState=OPEN, lastHost=961a81da9827,38709,1731954257723, regionLocation=961a81da9827,38709,1731954257723, openSeqNum=2 2024-11-18T18:27:27,725 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a244d0ab318eb4dd77bc5235828d9d51, regionState=OPEN, lastHost=961a81da9827,36379,1731954257826, regionLocation=961a81da9827,36379,1731954257826, openSeqNum=2 2024-11-18T18:27:27,725 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=012c277c99c564dd90e8e5f6850b6ce0, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=2 2024-11-18T18:27:27,725 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=140f5077d3b730b643b558ebe8a13abd, regionState=OPEN, lastHost=961a81da9827,38709,1731954257723, regionLocation=961a81da9827,38709,1731954257723, openSeqNum=2 2024-11-18T18:27:27,752 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e69ef2147d87d1b83dbcbb4705cbabe, regionState=OPEN, lastHost=961a81da9827,36379,1731954257826, regionLocation=961a81da9827,36379,1731954257826, openSeqNum=2 2024-11-18T18:27:27,752 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb8fa4293ea5e58c18dbc5b978355ca9, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=2 2024-11-18T18:27:27,752 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69ae79c6f9eeed559fa88701f773706c, regionState=OPEN, lastHost=961a81da9827,38709,1731954257723, regionLocation=961a81da9827,38709,1731954257723, openSeqNum=2 2024-11-18T18:27:27,753 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=75a13ed51395b1016b90c6b0d90a7d0a, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=2 2024-11-18T18:27:27,753 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aaf2435418703b2a31c1190af1560ba8, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=2 2024-11-18T18:27:27,753 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1ad74f79ba4d7fd1619c83a55106331, regionState=OPEN, lastHost=961a81da9827,36379,1731954257826, regionLocation=961a81da9827,36379,1731954257826, openSeqNum=2 2024-11-18T18:27:27,772 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=81f0db1bea1e11879dc1cafb06c0756d, regionState=OPEN, lastHost=961a81da9827,36379,1731954257826, regionLocation=961a81da9827,36379,1731954257826, openSeqNum=2 2024-11-18T18:27:27,772 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2a8fcaf8905dcde0537806fb16b9ff4c, regionState=OPEN, lastHost=961a81da9827,38709,1731954257723, regionLocation=961a81da9827,38709,1731954257723, openSeqNum=2 2024-11-18T18:27:27,773 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=579ee42a94597fdc1db9a4820a53ee64, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=2 2024-11-18T18:27:27,773 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=779798cd60db955ed78a4329cac08969, regionState=OPEN, lastHost=961a81da9827,36379,1731954257826, regionLocation=961a81da9827,36379,1731954257826, openSeqNum=2 2024-11-18T18:27:27,773 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=08ccc32db22fb75f5466ad774ad9e8ba, regionState=OPEN, lastHost=961a81da9827,38709,1731954257723, regionLocation=961a81da9827,38709,1731954257723, openSeqNum=2 2024-11-18T18:27:27,773 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-18T18:27:27,773 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731954507773 2024-11-18T18:27:27,773 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731954567773 2024-11-18T18:27:27,773 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 73 msec 2024-11-18T18:27:27,812 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 961a81da9827,36379,1731954257826, splitWal=true, meta=true, isMeta: false 2024-11-18T18:27:27,815 INFO [PEWorker-1 {}] master.SplitWALManager(105): 961a81da9827,36379,1731954257826 WAL count=1, meta=false 2024-11-18T18:27:27,815 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 961a81da9827%2C36379%2C1731954257826.1731954259295}] 2024-11-18T18:27:27,816 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-18T18:27:27,861 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=961a81da9827,42895,1731954257867 2024-11-18T18:27:27,866 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 961a81da9827%2C36379%2C1731954257826.1731954259295, worker=961a81da9827,42895,1731954257867}] 2024-11-18T18:27:28,058 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=63 2024-11-18T18:27:28,205 INFO [RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(299): Splitting hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/WALs/961a81da9827,36379,1731954257826-splitting/961a81da9827%2C36379%2C1731954257826.1731954259295, size=327.1 K (334920bytes) 2024-11-18T18:27:28,206 INFO [RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0-1 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/WALs/961a81da9827,36379,1731954257826-splitting/961a81da9827%2C36379%2C1731954257826.1731954259295 2024-11-18T18:27:28,207 INFO [RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0-1 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/WALs/961a81da9827,36379,1731954257826-splitting/961a81da9827%2C36379%2C1731954257826.1731954259295 after 0ms 2024-11-18T18:27:28,343 INFO [RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(310): Open hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/WALs/961a81da9827,36379,1731954257826-splitting/961a81da9827%2C36379%2C1731954257826.1731954259295 took 138ms 2024-11-18T18:27:28,490 DEBUG [RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 779798cd60db955ed78a4329cac08969: last_flushed_sequence_id: 18446744073709551615 2024-11-18T18:27:28,493 DEBUG [RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 81f0db1bea1e11879dc1cafb06c0756d: last_flushed_sequence_id: 18446744073709551615 2024-11-18T18:27:28,495 DEBUG [RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 43138a6a12c2efea7093064622bb3cd8: last_flushed_sequence_id: 18446744073709551615 2024-11-18T18:27:28,497 DEBUG [RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for d1ad74f79ba4d7fd1619c83a55106331: last_flushed_sequence_id: 18446744073709551615 2024-11-18T18:27:28,499 DEBUG [RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 5e69ef2147d87d1b83dbcbb4705cbabe: last_flushed_sequence_id: 18446744073709551615 2024-11-18T18:27:28,501 DEBUG [RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for bcf8695060a326b7d24615ba6cb932b1: last_flushed_sequence_id: 18446744073709551615 2024-11-18T18:27:28,511 DEBUG [RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for a244d0ab318eb4dd77bc5235828d9d51: last_flushed_sequence_id: 18446744073709551615 2024-11-18T18:27:28,512 DEBUG [RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for c3e2f3fb6516fcaaef4d0a2c8b63842d: last_flushed_sequence_id: 18446744073709551615 2024-11-18T18:27:28,514 DEBUG [RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 40a213dc6b2a0fd809c71612c437ab83: last_flushed_sequence_id: 18446744073709551615 2024-11-18T18:27:28,531 INFO [RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0-1-Writer-2 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp 2024-11-18T18:27:28,535 INFO [RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0-1-Writer-1 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp 2024-11-18T18:27:28,552 INFO [RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0-1-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp 2024-11-18T18:27:28,554 INFO [RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0-1-Writer-2 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/43138a6a12c2efea7093064622bb3cd8/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp 2024-11-18T18:27:28,575 INFO [RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0-1-Writer-2 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp 2024-11-18T18:27:28,586 INFO [RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0-1-Writer-1 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bcf8695060a326b7d24615ba6cb932b1/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp 2024-11-18T18:27:28,590 INFO [RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0-1-Writer-1 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp 2024-11-18T18:27:28,610 INFO [RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0-1-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/40a213dc6b2a0fd809c71612c437ab83/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp 2024-11-18T18:27:28,614 INFO [RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0-1-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp 2024-11-18T18:27:28,621 INFO [RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0-1-Writer-2 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c3e2f3fb6516fcaaef4d0a2c8b63842d/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp 2024-11-18T18:27:28,633 INFO [RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0-1-Writer-2 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp 2024-11-18T18:27:28,637 INFO [RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0-1-Writer-1 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a244d0ab318eb4dd77bc5235828d9d51/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp 2024-11-18T18:27:28,643 INFO [RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0-1-Writer-1 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp 2024-11-18T18:27:28,700 INFO [RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0-1-Writer-2 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/81f0db1bea1e11879dc1cafb06c0756d/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp 2024-11-18T18:27:28,700 INFO [RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0-1-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5e69ef2147d87d1b83dbcbb4705cbabe/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp 2024-11-18T18:27:28,702 INFO [RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0-1-Writer-1 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/779798cd60db955ed78a4329cac08969/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp 2024-11-18T18:27:28,704 INFO [RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0-1-Writer-2 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp 2024-11-18T18:27:28,743 INFO [RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0-1-Writer-2 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d1ad74f79ba4d7fd1619c83a55106331/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp 2024-11-18T18:27:28,759 DEBUG [RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/WALs/961a81da9827,36379,1731954257826-splitting/961a81da9827%2C36379%2C1731954257826.1731954259295 so closing down 2024-11-18T18:27:28,759 DEBUG [RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-18T18:27:28,760 INFO [RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(145): 3 split writer threads finished 2024-11-18T18:27:29,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741869_1045 (size=36767) 2024-11-18T18:27:29,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741870_1046 (size=36767) 2024-11-18T18:27:29,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741869_1045 (size=36767) 2024-11-18T18:27:29,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741870_1046 (size=36767) 2024-11-18T18:27:29,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741869_1045 (size=36767) 2024-11-18T18:27:29,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741870_1046 (size=36767) 2024-11-18T18:27:29,192 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/40a213dc6b2a0fd809c71612c437ab83/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-18T18:27:29,193 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/81f0db1bea1e11879dc1cafb06c0756d/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp (wrote 1 edits, skipped 0 edits in 1 ms) 2024-11-18T18:27:29,194 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/40a213dc6b2a0fd809c71612c437ab83/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp to hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/40a213dc6b2a0fd809c71612c437ab83/recovered.edits/0000000000000000004 2024-11-18T18:27:29,195 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/81f0db1bea1e11879dc1cafb06c0756d/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp to hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/81f0db1bea1e11879dc1cafb06c0756d/recovered.edits/0000000000000000004 2024-11-18T18:27:29,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741871_1047 (size=36767) 2024-11-18T18:27:29,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741871_1047 (size=36767) 2024-11-18T18:27:29,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741871_1047 (size=36767) 2024-11-18T18:27:29,449 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/779798cd60db955ed78a4329cac08969/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-18T18:27:29,467 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/779798cd60db955ed78a4329cac08969/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp to hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/779798cd60db955ed78a4329cac08969/recovered.edits/0000000000000000004 2024-11-18T18:27:29,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741873_1049 (size=36767) 2024-11-18T18:27:29,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741872_1048 (size=36767) 2024-11-18T18:27:29,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741872_1048 (size=36767) 2024-11-18T18:27:29,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741873_1049 (size=36767) 2024-11-18T18:27:29,591 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a244d0ab318eb4dd77bc5235828d9d51/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-18T18:27:29,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741873_1049 (size=36767) 2024-11-18T18:27:29,597 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a244d0ab318eb4dd77bc5235828d9d51/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp to hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a244d0ab318eb4dd77bc5235828d9d51/recovered.edits/0000000000000000004 2024-11-18T18:27:29,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741872_1048 (size=36767) 2024-11-18T18:27:29,643 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c3e2f3fb6516fcaaef4d0a2c8b63842d/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp (wrote 1 edits, skipped 0 edits in 7 ms) 2024-11-18T18:27:29,646 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c3e2f3fb6516fcaaef4d0a2c8b63842d/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp to hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c3e2f3fb6516fcaaef4d0a2c8b63842d/recovered.edits/0000000000000000004 2024-11-18T18:27:29,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741874_1050 (size=36767) 2024-11-18T18:27:29,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741874_1050 (size=36767) 2024-11-18T18:27:29,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741874_1050 (size=36767) 2024-11-18T18:27:29,714 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d1ad74f79ba4d7fd1619c83a55106331/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-18T18:27:29,717 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d1ad74f79ba4d7fd1619c83a55106331/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp to hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d1ad74f79ba4d7fd1619c83a55106331/recovered.edits/0000000000000000004 2024-11-18T18:27:30,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741875_1051 (size=36767) 2024-11-18T18:27:30,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741875_1051 (size=36767) 2024-11-18T18:27:30,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741875_1051 (size=36767) 2024-11-18T18:27:30,229 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5e69ef2147d87d1b83dbcbb4705cbabe/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp (wrote 1 edits, skipped 0 edits in 1 ms) 2024-11-18T18:27:30,231 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5e69ef2147d87d1b83dbcbb4705cbabe/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp to hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5e69ef2147d87d1b83dbcbb4705cbabe/recovered.edits/0000000000000000004 2024-11-18T18:27:30,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741876_1052 (size=36767) 2024-11-18T18:27:30,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741876_1052 (size=36767) 2024-11-18T18:27:30,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741876_1052 (size=36767) 2024-11-18T18:27:30,240 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/43138a6a12c2efea7093064622bb3cd8/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp (wrote 1 edits, skipped 0 edits in 1 ms) 2024-11-18T18:27:30,318 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/43138a6a12c2efea7093064622bb3cd8/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp to hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/43138a6a12c2efea7093064622bb3cd8/recovered.edits/0000000000000000004 2024-11-18T18:27:30,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741877_1053 (size=36767) 2024-11-18T18:27:30,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741877_1053 (size=36767) 2024-11-18T18:27:30,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741877_1053 (size=36767) 2024-11-18T18:27:30,386 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bcf8695060a326b7d24615ba6cb932b1/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-18T18:27:30,388 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bcf8695060a326b7d24615ba6cb932b1/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp to hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bcf8695060a326b7d24615ba6cb932b1/recovered.edits/0000000000000000004 2024-11-18T18:27:30,389 INFO [RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(425): Processed 18 edits across 9 Regions in 1904 ms; skipped=9; WAL=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/WALs/961a81da9827,36379,1731954257826-splitting/961a81da9827%2C36379%2C1731954257826.1731954259295, size=327.1 K, length=334920, corrupted=false, cancelled=false 2024-11-18T18:27:30,389 DEBUG [RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(428): Completed split of hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/WALs/961a81da9827,36379,1731954257826-splitting/961a81da9827%2C36379%2C1731954257826.1731954259295, journal: Splitting hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/WALs/961a81da9827,36379,1731954257826-splitting/961a81da9827%2C36379%2C1731954257826.1731954259295, size=327.1 K (334920bytes) at 1731954448205Creating recovered edits writer path=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/43138a6a12c2efea7093064622bb3cd8/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp at 1731954448554 (+349 ms)Creating recovered edits writer path=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bcf8695060a326b7d24615ba6cb932b1/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp at 1731954448586 (+32 ms)Creating recovered edits writer path=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/40a213dc6b2a0fd809c71612c437ab83/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp at 1731954448610 (+24 ms)Creating recovered edits writer path=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c3e2f3fb6516fcaaef4d0a2c8b63842d/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp at 1731954448621 (+11 ms)Creating recovered edits writer path=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a244d0ab318eb4dd77bc5235828d9d51/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp at 1731954448638 (+17 ms)Creating recovered edits writer path=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/81f0db1bea1e11879dc1cafb06c0756d/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp at 1731954448700 (+62 ms)Creating recovered edits writer path=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5e69ef2147d87d1b83dbcbb4705cbabe/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp at 1731954448700Creating recovered edits writer path=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/779798cd60db955ed78a4329cac08969/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp at 1731954448702 (+2 ms)Creating recovered edits writer path=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d1ad74f79ba4d7fd1619c83a55106331/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp at 1731954448743 (+41 ms)Split 9 edits, skipped 9 edits. at 1731954448759 (+16 ms)Finishing writing output for hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/WALs/961a81da9827,36379,1731954257826-splitting/961a81da9827%2C36379%2C1731954257826.1731954259295 so closing down at 17319544487593 split writer threads finished at 1731954448760 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/40a213dc6b2a0fd809c71612c437ab83/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731954449192 (+432 ms)Closed recovered edits writer path=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/81f0db1bea1e11879dc1cafb06c0756d/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp (wrote 1 edits, skipped 0 edits in 1 ms) at 1731954449193 (+1 ms)Rename recovered edits hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/40a213dc6b2a0fd809c71612c437ab83/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp to hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/40a213dc6b2a0fd809c71612c437ab83/recovered.edits/0000000000000000004 at 1731954449194 (+1 ms)Rename recovered edits hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/81f0db1bea1e11879dc1cafb06c0756d/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp to hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/81f0db1bea1e11879dc1cafb06c0756d/recovered.edits/0000000000000000004 at 1731954449195 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/779798cd60db955ed78a4329cac08969/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731954449449 (+254 ms)Rename recovered edits hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/779798cd60db955ed78a4329cac08969/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp to hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/779798cd60db955ed78a4329cac08969/recovered.edits/0000000000000000004 at 1731954449467 (+18 ms)Closed recovered edits writer path=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a244d0ab318eb4dd77bc5235828d9d51/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731954449591 (+124 ms)Rename recovered edits hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a244d0ab318eb4dd77bc5235828d9d51/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp to hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a244d0ab318eb4dd77bc5235828d9d51/recovered.edits/0000000000000000004 at 1731954449597 (+6 ms)Closed recovered edits writer path=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c3e2f3fb6516fcaaef4d0a2c8b63842d/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp (wrote 1 edits, skipped 0 edits in 7 ms) at 1731954449643 (+46 ms)Rename recovered edits hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c3e2f3fb6516fcaaef4d0a2c8b63842d/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp to hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c3e2f3fb6516fcaaef4d0a2c8b63842d/recovered.edits/0000000000000000004 at 1731954449646 (+3 ms)Closed recovered edits writer path=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d1ad74f79ba4d7fd1619c83a55106331/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731954449715 (+69 ms)Rename recovered edits hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d1ad74f79ba4d7fd1619c83a55106331/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp to hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d1ad74f79ba4d7fd1619c83a55106331/recovered.edits/0000000000000000004 at 1731954449717 (+2 ms)Closed recovered edits writer path=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5e69ef2147d87d1b83dbcbb4705cbabe/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp (wrote 1 edits, skipped 0 edits in 1 ms) at 1731954450229 (+512 ms)Rename recovered edits hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5e69ef2147d87d1b83dbcbb4705cbabe/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp to hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5e69ef2147d87d1b83dbcbb4705cbabe/recovered.edits/0000000000000000004 at 1731954450231 (+2 ms)Closed recovered edits writer path=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/43138a6a12c2efea7093064622bb3cd8/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp (wrote 1 edits, skipped 0 edits in 1 ms) at 1731954450240 (+9 ms)Rename recovered edits hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/43138a6a12c2efea7093064622bb3cd8/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp to hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/43138a6a12c2efea7093064622bb3cd8/recovered.edits/0000000000000000004 at 1731954450319 (+79 ms)Closed recovered edits writer path=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bcf8695060a326b7d24615ba6cb932b1/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731954450386 (+67 ms)Rename recovered edits hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bcf8695060a326b7d24615ba6cb932b1/recovered.edits/0000000000000000004-961a81da9827%2C36379%2C1731954257826.1731954259295.temp to hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bcf8695060a326b7d24615ba6cb932b1/recovered.edits/0000000000000000004 at 1731954450388 (+2 ms)Processed 18 edits across 9 Regions in 1904 ms; skipped=9; WAL=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/WALs/961a81da9827,36379,1731954257826-splitting/961a81da9827%2C36379%2C1731954257826.1731954259295, size=327.1 K, length=334920, corrupted=false, cancelled=false at 1731954450389 (+1 ms) 2024-11-18T18:27:30,390 DEBUG [RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0-1 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/WALs/961a81da9827,36379,1731954257826-splitting/961a81da9827%2C36379%2C1731954257826.1731954259295 2024-11-18T18:27:30,390 DEBUG [RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0-1 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-18T18:27:30,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33089 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-18T18:27:30,399 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/WALs/961a81da9827,36379,1731954257826-splitting/961a81da9827%2C36379%2C1731954257826.1731954259295 to hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/oldWALs 2024-11-18T18:27:30,404 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-18T18:27:30,404 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 961a81da9827%2C36379%2C1731954257826.1731954259295, worker=961a81da9827,42895,1731954257867 in 2.5330 sec 2024-11-18T18:27:30,406 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=961a81da9827,42895,1731954257867 2024-11-18T18:27:30,411 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=57 2024-11-18T18:27:30,411 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 961a81da9827%2C36379%2C1731954257826.1731954259295, worker=961a81da9827,42895,1731954257867 in 2.5910 sec 2024-11-18T18:27:30,414 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/WALs/961a81da9827,36379,1731954257826-splitting dir is empty, no logs to split. 2024-11-18T18:27:30,414 INFO [PEWorker-1 {}] master.SplitWALManager(105): 961a81da9827,36379,1731954257826 WAL count=0, meta=false 2024-11-18T18:27:30,415 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 961a81da9827,36379,1731954257826 WAL splitting is done? wals=0, meta=false 2024-11-18T18:27:30,416 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-18T18:27:30,416 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN, hasLock=true; ServerCrashProcedure 961a81da9827,36379,1731954257826, splitWal=true, meta=true 2024-11-18T18:27:30,416 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T18:27:30,538 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 6 exec state=pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN, hasLock=true; ServerCrashProcedure 961a81da9827,36379,1731954257826, splitWal=true, meta=true 2024-11-18T18:27:30,538 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-18T18:27:30,538 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T18:27:30,538 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-18T18:27:30,538 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33089 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T18:27:30,539 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-18T18:27:30,538 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN, hasLock=true; ServerCrashProcedure 961a81da9827,36379,1731954257826, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T18:27:30,540 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-18T18:27:30,540 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-18T18:27:30,540 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-18T18:27:30,540 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33089 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T18:27:30,540 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-18T18:27:30,541 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-18T18:27:30,543 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-18T18:27:30,543 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-18T18:27:30,543 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-18T18:27:30,543 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T18:27:30,543 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T18:27:30,543 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-18T18:27:30,543 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-18T18:27:30,543 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-18T18:27:30,544 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-18T18:27:30,546 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_DELETE_SPLIT_WALS_DIR, hasLock=false; ServerCrashProcedure 961a81da9827,36379,1731954257826, splitWal=true, meta=true 2024-11-18T18:27:30,546 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 961a81da9827%2C36379%2C1731954257826.meta.1731954259788.meta, worker=961a81da9827,42895,1731954257867 2024-11-18T18:27:30,546 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 961a81da9827%2C36379%2C1731954257826.meta.1731954259788.meta, worker=961a81da9827,42895,1731954257867 2024-11-18T18:27:30,546 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T18:27:30,546 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=961a81da9827,42895,1731954257867 2024-11-18T18:27:30,546 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 961a81da9827%2C36379%2C1731954257826.1731954259295, worker=961a81da9827,42895,1731954257867 2024-11-18T18:27:30,546 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 961a81da9827%2C36379%2C1731954257826.1731954259295, worker=961a81da9827,42895,1731954257867 2024-11-18T18:27:30,546 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_DELETE_SPLIT_WALS_DIR, hasLock=false; ServerCrashProcedure 961a81da9827,36379,1731954257826, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-18T18:27:30,546 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 961a81da9827%2C36379%2C1731954257826.meta.1731954259788.meta, worker=961a81da9827,42895,1731954257867 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T18:27:30,546 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 961a81da9827%2C36379%2C1731954257826.meta.1731954259788.meta, worker=961a81da9827,42895,1731954257867 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T18:27:30,546 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T18:27:30,546 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=961a81da9827,42895,1731954257867 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T18:27:30,546 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 961a81da9827%2C36379%2C1731954257826.1731954259295, worker=961a81da9827,42895,1731954257867 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T18:27:30,546 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 961a81da9827%2C36379%2C1731954257826.1731954259295, worker=961a81da9827,42895,1731954257867 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T18:27:30,547 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-18T18:27:30,643 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33089 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T18:27:30,645 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33089 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T18:27:30,690 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=61 2024-11-18T18:27:30,692 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=961a81da9827,42895,1731954257867, table=hbase:meta, region=1588230740 2024-11-18T18:27:30,692 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 961a81da9827,42895,1731954257867, state=OPEN 2024-11-18T18:27:30,695 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33089-0x10107179d9f0000, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T18:27:30,695 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42895-0x10107179d9f0003, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T18:27:30,695 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:27:30,695 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:27:30,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38709-0x10107179d9f0001, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T18:27:30,696 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:27:30,717 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731954480717 2024-11-18T18:27:30,718 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-18T18:27:30,725 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8bb7ae0747eda2290799ca8fa83f9ede, regionState=OPEN, lastHost=961a81da9827,38709,1731954257723, regionLocation=961a81da9827,38709,1731954257723, openSeqNum=2 2024-11-18T18:27:30,725 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22971deb0db38b7cccde5c3bc0868f22, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=2 2024-11-18T18:27:30,725 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43138a6a12c2efea7093064622bb3cd8, regionState=OPEN, lastHost=961a81da9827,36379,1731954257826, regionLocation=961a81da9827,36379,1731954257826, openSeqNum=2 2024-11-18T18:27:30,725 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb099f0975ebdb618ef8e3b2c263bbca, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=2 2024-11-18T18:27:30,725 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bcf8695060a326b7d24615ba6cb932b1, regionState=OPEN, lastHost=961a81da9827,36379,1731954257826, regionLocation=961a81da9827,36379,1731954257826, openSeqNum=2 2024-11-18T18:27:30,726 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb91ff28ef3c18f0dc0d3f26aac68eca, regionState=OPEN, lastHost=961a81da9827,38709,1731954257723, regionLocation=961a81da9827,38709,1731954257723, openSeqNum=2 2024-11-18T18:27:30,726 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8b4dd2a6d8dd59085c547b55aac0de6, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=2 2024-11-18T18:27:30,726 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40a213dc6b2a0fd809c71612c437ab83, regionState=OPEN, lastHost=961a81da9827,36379,1731954257826, regionLocation=961a81da9827,36379,1731954257826, openSeqNum=2 2024-11-18T18:27:30,726 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f39571a93e25091090ea0831d54936b5, regionState=OPEN, lastHost=961a81da9827,38709,1731954257723, regionLocation=961a81da9827,38709,1731954257723, openSeqNum=2 2024-11-18T18:27:30,726 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97a938e093c14701871e85a1fbb6e07d, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=2 2024-11-18T18:27:30,726 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c3e2f3fb6516fcaaef4d0a2c8b63842d, regionState=OPEN, lastHost=961a81da9827,36379,1731954257826, regionLocation=961a81da9827,36379,1731954257826, openSeqNum=2 2024-11-18T18:27:30,726 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97377a91d35f8b242120f603c456c41e, regionState=OPEN, lastHost=961a81da9827,38709,1731954257723, regionLocation=961a81da9827,38709,1731954257723, openSeqNum=2 2024-11-18T18:27:30,726 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a244d0ab318eb4dd77bc5235828d9d51, regionState=OPEN, lastHost=961a81da9827,36379,1731954257826, regionLocation=961a81da9827,36379,1731954257826, openSeqNum=2 2024-11-18T18:27:30,727 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=012c277c99c564dd90e8e5f6850b6ce0, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=2 2024-11-18T18:27:30,727 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=140f5077d3b730b643b558ebe8a13abd, regionState=OPEN, lastHost=961a81da9827,38709,1731954257723, regionLocation=961a81da9827,38709,1731954257723, openSeqNum=2 2024-11-18T18:27:30,727 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e69ef2147d87d1b83dbcbb4705cbabe, regionState=OPEN, lastHost=961a81da9827,36379,1731954257826, regionLocation=961a81da9827,36379,1731954257826, openSeqNum=2 2024-11-18T18:27:30,727 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb8fa4293ea5e58c18dbc5b978355ca9, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=2 2024-11-18T18:27:30,727 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69ae79c6f9eeed559fa88701f773706c, regionState=OPEN, lastHost=961a81da9827,38709,1731954257723, regionLocation=961a81da9827,38709,1731954257723, openSeqNum=2 2024-11-18T18:27:30,727 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=75a13ed51395b1016b90c6b0d90a7d0a, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=2 2024-11-18T18:27:30,727 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aaf2435418703b2a31c1190af1560ba8, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=2 2024-11-18T18:27:30,727 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1ad74f79ba4d7fd1619c83a55106331, regionState=OPEN, lastHost=961a81da9827,36379,1731954257826, regionLocation=961a81da9827,36379,1731954257826, openSeqNum=2 2024-11-18T18:27:30,727 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=81f0db1bea1e11879dc1cafb06c0756d, regionState=OPEN, lastHost=961a81da9827,36379,1731954257826, regionLocation=961a81da9827,36379,1731954257826, openSeqNum=2 2024-11-18T18:27:30,728 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2a8fcaf8905dcde0537806fb16b9ff4c, regionState=OPEN, lastHost=961a81da9827,38709,1731954257723, regionLocation=961a81da9827,38709,1731954257723, openSeqNum=2 2024-11-18T18:27:30,728 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=579ee42a94597fdc1db9a4820a53ee64, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=2 2024-11-18T18:27:30,728 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=779798cd60db955ed78a4329cac08969, regionState=OPEN, lastHost=961a81da9827,36379,1731954257826, regionLocation=961a81da9827,36379,1731954257826, openSeqNum=2 2024-11-18T18:27:30,728 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=08ccc32db22fb75f5466ad774ad9e8ba, regionState=OPEN, lastHost=961a81da9827,38709,1731954257723, regionLocation=961a81da9827,38709,1731954257723, openSeqNum=2 2024-11-18T18:27:30,728 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-18T18:27:30,728 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731954510728 2024-11-18T18:27:30,728 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731954570728 2024-11-18T18:27:30,728 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-18T18:27:30,730 INFO [PEWorker-1 {}] master.MasterWalManager(331): Log dir for server 961a81da9827,36379,1731954257826 does not exist 2024-11-18T18:27:30,730 INFO [PEWorker-1 {}] master.SplitWALManager(105): 961a81da9827,36379,1731954257826 WAL count=0, meta=false 2024-11-18T18:27:30,730 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 961a81da9827,36379,1731954257826 WAL splitting is done? wals=0, meta=false 2024-11-18T18:27:30,732 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 961a81da9827,36379,1731954257826 failed, ignore...File hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/WALs/961a81da9827,36379,1731954257826-splitting does not exist. 2024-11-18T18:27:30,732 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-18T18:27:30,864 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5e69ef2147d87d1b83dbcbb4705cbabe, ASSIGN}, {pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=bcf8695060a326b7d24615ba6cb932b1, ASSIGN}, {pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=43138a6a12c2efea7093064622bb3cd8, ASSIGN}, {pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c3e2f3fb6516fcaaef4d0a2c8b63842d, ASSIGN}, {pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=40a213dc6b2a0fd809c71612c437ab83, ASSIGN}, {pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d1ad74f79ba4d7fd1619c83a55106331, ASSIGN}, {pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a244d0ab318eb4dd77bc5235828d9d51, ASSIGN}, {pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=81f0db1bea1e11879dc1cafb06c0756d, ASSIGN}, {pid=72, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=779798cd60db955ed78a4329cac08969, ASSIGN}] 2024-11-18T18:27:30,864 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-18T18:27:30,864 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_FINISH, hasLock=true; ServerCrashProcedure 961a81da9827,36379,1731954257826, splitWal=true, meta=true 2024-11-18T18:27:30,864 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T18:27:30,865 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_FINISH, hasLock=true; ServerCrashProcedure 961a81da9827,36379,1731954257826, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T18:27:30,978 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 7 exec state=pid=57, state=WAITING:SERVER_CRASH_FINISH, hasLock=true; ServerCrashProcedure 961a81da9827,36379,1731954257826, splitWal=true, meta=true 2024-11-18T18:27:30,979 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-18T18:27:30,979 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T18:27:30,979 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-18T18:27:30,979 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-18T18:27:30,979 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-18T18:27:30,980 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-18T18:27:30,980 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-18T18:27:30,980 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-18T18:27:30,981 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-18T18:27:30,983 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-18T18:27:30,983 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-18T18:27:30,983 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-18T18:27:30,983 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T18:27:30,983 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T18:27:30,983 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-18T18:27:30,983 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-18T18:27:30,984 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-18T18:27:31,001 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-18T18:27:31,003 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN, hasLock=false; ServerCrashProcedure 961a81da9827,36379,1731954257826, splitWal=true, meta=true 2024-11-18T18:27:31,003 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 961a81da9827%2C36379%2C1731954257826.meta.1731954259788.meta, worker=961a81da9827,42895,1731954257867 2024-11-18T18:27:31,003 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 961a81da9827%2C36379%2C1731954257826.meta.1731954259788.meta, worker=961a81da9827,42895,1731954257867 2024-11-18T18:27:31,003 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T18:27:31,003 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=961a81da9827,42895,1731954257867 2024-11-18T18:27:31,003 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 961a81da9827%2C36379%2C1731954257826.1731954259295, worker=961a81da9827,42895,1731954257867 2024-11-18T18:27:31,003 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 961a81da9827%2C36379%2C1731954257826.1731954259295, worker=961a81da9827,42895,1731954257867 2024-11-18T18:27:31,003 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN, hasLock=false; ServerCrashProcedure 961a81da9827,36379,1731954257826, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-18T18:27:31,003 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 961a81da9827%2C36379%2C1731954257826.meta.1731954259788.meta, worker=961a81da9827,42895,1731954257867 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T18:27:31,003 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 961a81da9827%2C36379%2C1731954257826.meta.1731954259788.meta, worker=961a81da9827,42895,1731954257867 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T18:27:31,003 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T18:27:31,003 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=961a81da9827,42895,1731954257867 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T18:27:31,003 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 961a81da9827%2C36379%2C1731954257826.1731954259295, worker=961a81da9827,42895,1731954257867 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T18:27:31,003 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 961a81da9827%2C36379%2C1731954257826.1731954259295, worker=961a81da9827,42895,1731954257867 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T18:27:31,004 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 22 msec 2024-11-18T18:27:31,013 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=61 2024-11-18T18:27:31,013 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=961a81da9827,42895,1731954257867, table=hbase:meta, region=1588230740 2024-11-18T18:27:31,014 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 961a81da9827,42895,1731954257867, state=OPEN 2024-11-18T18:27:31,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38709-0x10107179d9f0001, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T18:27:31,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42895-0x10107179d9f0003, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T18:27:31,016 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:27:31,016 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:27:31,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33089-0x10107179d9f0000, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T18:27:31,017 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:27:31,045 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731954481045 2024-11-18T18:27:31,046 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-18T18:27:31,055 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8bb7ae0747eda2290799ca8fa83f9ede, regionState=OPEN, lastHost=961a81da9827,38709,1731954257723, regionLocation=961a81da9827,38709,1731954257723, openSeqNum=2 2024-11-18T18:27:31,055 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22971deb0db38b7cccde5c3bc0868f22, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=2 2024-11-18T18:27:31,055 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43138a6a12c2efea7093064622bb3cd8, regionState=OPEN, lastHost=961a81da9827,36379,1731954257826, regionLocation=961a81da9827,36379,1731954257826, openSeqNum=2 2024-11-18T18:27:31,055 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb099f0975ebdb618ef8e3b2c263bbca, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=2 2024-11-18T18:27:31,056 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bcf8695060a326b7d24615ba6cb932b1, regionState=OPEN, lastHost=961a81da9827,36379,1731954257826, regionLocation=961a81da9827,36379,1731954257826, openSeqNum=2 2024-11-18T18:27:31,056 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb91ff28ef3c18f0dc0d3f26aac68eca, regionState=OPEN, lastHost=961a81da9827,38709,1731954257723, regionLocation=961a81da9827,38709,1731954257723, openSeqNum=2 2024-11-18T18:27:31,056 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8b4dd2a6d8dd59085c547b55aac0de6, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=2 2024-11-18T18:27:31,056 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40a213dc6b2a0fd809c71612c437ab83, regionState=OPEN, lastHost=961a81da9827,36379,1731954257826, regionLocation=961a81da9827,36379,1731954257826, openSeqNum=2 2024-11-18T18:27:31,056 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f39571a93e25091090ea0831d54936b5, regionState=OPEN, lastHost=961a81da9827,38709,1731954257723, regionLocation=961a81da9827,38709,1731954257723, openSeqNum=2 2024-11-18T18:27:31,056 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97a938e093c14701871e85a1fbb6e07d, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=2 2024-11-18T18:27:31,056 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c3e2f3fb6516fcaaef4d0a2c8b63842d, regionState=OPEN, lastHost=961a81da9827,36379,1731954257826, regionLocation=961a81da9827,36379,1731954257826, openSeqNum=2 2024-11-18T18:27:31,057 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97377a91d35f8b242120f603c456c41e, regionState=OPEN, lastHost=961a81da9827,38709,1731954257723, regionLocation=961a81da9827,38709,1731954257723, openSeqNum=2 2024-11-18T18:27:31,057 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a244d0ab318eb4dd77bc5235828d9d51, regionState=OPEN, lastHost=961a81da9827,36379,1731954257826, regionLocation=961a81da9827,36379,1731954257826, openSeqNum=2 2024-11-18T18:27:31,057 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=012c277c99c564dd90e8e5f6850b6ce0, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=2 2024-11-18T18:27:31,057 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=140f5077d3b730b643b558ebe8a13abd, regionState=OPEN, lastHost=961a81da9827,38709,1731954257723, regionLocation=961a81da9827,38709,1731954257723, openSeqNum=2 2024-11-18T18:27:31,057 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e69ef2147d87d1b83dbcbb4705cbabe, regionState=OPEN, lastHost=961a81da9827,36379,1731954257826, regionLocation=961a81da9827,36379,1731954257826, openSeqNum=2 2024-11-18T18:27:31,057 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb8fa4293ea5e58c18dbc5b978355ca9, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=2 2024-11-18T18:27:31,058 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69ae79c6f9eeed559fa88701f773706c, regionState=OPEN, lastHost=961a81da9827,38709,1731954257723, regionLocation=961a81da9827,38709,1731954257723, openSeqNum=2 2024-11-18T18:27:31,058 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=75a13ed51395b1016b90c6b0d90a7d0a, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=2 2024-11-18T18:27:31,058 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aaf2435418703b2a31c1190af1560ba8, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=2 2024-11-18T18:27:31,058 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1ad74f79ba4d7fd1619c83a55106331, regionState=OPEN, lastHost=961a81da9827,36379,1731954257826, regionLocation=961a81da9827,36379,1731954257826, openSeqNum=2 2024-11-18T18:27:31,058 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=81f0db1bea1e11879dc1cafb06c0756d, regionState=OPEN, lastHost=961a81da9827,36379,1731954257826, regionLocation=961a81da9827,36379,1731954257826, openSeqNum=2 2024-11-18T18:27:31,058 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2a8fcaf8905dcde0537806fb16b9ff4c, regionState=OPEN, lastHost=961a81da9827,38709,1731954257723, regionLocation=961a81da9827,38709,1731954257723, openSeqNum=2 2024-11-18T18:27:31,059 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=579ee42a94597fdc1db9a4820a53ee64, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=2 2024-11-18T18:27:31,059 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=779798cd60db955ed78a4329cac08969, regionState=OPEN, lastHost=961a81da9827,36379,1731954257826, regionLocation=961a81da9827,36379,1731954257826, openSeqNum=2 2024-11-18T18:27:31,059 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=08ccc32db22fb75f5466ad774ad9e8ba, regionState=OPEN, lastHost=961a81da9827,38709,1731954257723, regionLocation=961a81da9827,38709,1731954257723, openSeqNum=2 2024-11-18T18:27:31,059 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-18T18:27:31,059 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731954511059 2024-11-18T18:27:31,059 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731954571059 2024-11-18T18:27:31,059 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 13 msec 2024-11-18T18:27:31,060 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5e69ef2147d87d1b83dbcbb4705cbabe, ASSIGN}, {pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=bcf8695060a326b7d24615ba6cb932b1, ASSIGN}, {pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=43138a6a12c2efea7093064622bb3cd8, ASSIGN}, {pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c3e2f3fb6516fcaaef4d0a2c8b63842d, ASSIGN}, {pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=40a213dc6b2a0fd809c71612c437ab83, ASSIGN}, {pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d1ad74f79ba4d7fd1619c83a55106331, ASSIGN}, {pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a244d0ab318eb4dd77bc5235828d9d51, ASSIGN}, {pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=81f0db1bea1e11879dc1cafb06c0756d, ASSIGN}, {pid=72, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=779798cd60db955ed78a4329cac08969, ASSIGN}] 2024-11-18T18:27:31,060 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-18T18:27:31,063 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=779798cd60db955ed78a4329cac08969, ASSIGN 2024-11-18T18:27:31,064 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=72, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=779798cd60db955ed78a4329cac08969, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-18T18:27:31,065 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=81f0db1bea1e11879dc1cafb06c0756d, ASSIGN 2024-11-18T18:27:31,067 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=81f0db1bea1e11879dc1cafb06c0756d, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-18T18:27:31,068 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a244d0ab318eb4dd77bc5235828d9d51, ASSIGN 2024-11-18T18:27:31,069 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a244d0ab318eb4dd77bc5235828d9d51, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-18T18:27:31,071 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d1ad74f79ba4d7fd1619c83a55106331, ASSIGN 2024-11-18T18:27:31,072 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d1ad74f79ba4d7fd1619c83a55106331, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-18T18:27:31,073 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=40a213dc6b2a0fd809c71612c437ab83, ASSIGN 2024-11-18T18:27:31,074 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=40a213dc6b2a0fd809c71612c437ab83, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-18T18:27:31,075 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c3e2f3fb6516fcaaef4d0a2c8b63842d, ASSIGN 2024-11-18T18:27:31,076 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c3e2f3fb6516fcaaef4d0a2c8b63842d, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-18T18:27:31,077 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=43138a6a12c2efea7093064622bb3cd8, ASSIGN 2024-11-18T18:27:31,079 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=43138a6a12c2efea7093064622bb3cd8, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-18T18:27:31,080 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=bcf8695060a326b7d24615ba6cb932b1, ASSIGN 2024-11-18T18:27:31,081 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=bcf8695060a326b7d24615ba6cb932b1, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-18T18:27:31,082 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5e69ef2147d87d1b83dbcbb4705cbabe, ASSIGN 2024-11-18T18:27:31,083 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5e69ef2147d87d1b83dbcbb4705cbabe, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-18T18:27:31,215 DEBUG [961a81da9827:33089 {}] balancer.BalancerClusterState(204): Hosts are {961a81da9827=0} racks are {/default-rack=0} 2024-11-18T18:27:31,216 DEBUG [961a81da9827:33089 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-18T18:27:31,216 DEBUG [961a81da9827:33089 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-18T18:27:31,216 DEBUG [961a81da9827:33089 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-18T18:27:31,216 DEBUG [961a81da9827:33089 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-18T18:27:31,216 INFO [961a81da9827:33089 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-18T18:27:31,216 INFO [961a81da9827:33089 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-18T18:27:31,216 DEBUG [961a81da9827:33089 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-18T18:27:31,225 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=779798cd60db955ed78a4329cac08969, regionState=OPENING, regionLocation=961a81da9827,42895,1731954257867 2024-11-18T18:27:31,226 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=d1ad74f79ba4d7fd1619c83a55106331, regionState=OPENING, regionLocation=961a81da9827,42895,1731954257867 2024-11-18T18:27:31,227 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=a244d0ab318eb4dd77bc5235828d9d51, regionState=OPENING, regionLocation=961a81da9827,42895,1731954257867 2024-11-18T18:27:31,227 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=40a213dc6b2a0fd809c71612c437ab83, regionState=OPENING, regionLocation=961a81da9827,42895,1731954257867 2024-11-18T18:27:31,228 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=43138a6a12c2efea7093064622bb3cd8, regionState=OPENING, regionLocation=961a81da9827,42895,1731954257867 2024-11-18T18:27:31,228 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=81f0db1bea1e11879dc1cafb06c0756d, regionState=OPENING, regionLocation=961a81da9827,38709,1731954257723 2024-11-18T18:27:31,229 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=72, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=779798cd60db955ed78a4329cac08969, ASSIGN because future has completed 2024-11-18T18:27:31,229 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=5e69ef2147d87d1b83dbcbb4705cbabe, regionState=OPENING, regionLocation=961a81da9827,38709,1731954257723 2024-11-18T18:27:31,230 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=c3e2f3fb6516fcaaef4d0a2c8b63842d, regionState=OPENING, regionLocation=961a81da9827,38709,1731954257723 2024-11-18T18:27:31,230 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=65 updating hbase:meta row=bcf8695060a326b7d24615ba6cb932b1, regionState=OPENING, regionLocation=961a81da9827,38709,1731954257723 2024-11-18T18:27:31,230 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d1ad74f79ba4d7fd1619c83a55106331, ASSIGN because future has completed 2024-11-18T18:27:31,231 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 779798cd60db955ed78a4329cac08969, server=961a81da9827,42895,1731954257867}] 2024-11-18T18:27:31,233 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a244d0ab318eb4dd77bc5235828d9d51, ASSIGN because future has completed 2024-11-18T18:27:31,236 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=40a213dc6b2a0fd809c71612c437ab83, ASSIGN because future has completed 2024-11-18T18:27:31,362 INFO [AsyncFSWAL-0-hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93-prefix:961a81da9827,42895,1731954257867.meta {}] wal.AbstractFSWAL(1368): Slow sync cost: 125 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41713,DS-c75b1747-362b-431f-b47d-543a4f882cdb,DISK], DatanodeInfoWithStorage[127.0.0.1:34047,DS-1029886b-6b51-4ae4-81a3-156afc7de901,DISK], DatanodeInfoWithStorage[127.0.0.1:44191,DS-39ff43bf-c18b-4b86-b37c-eca99a433383,DISK]] 2024-11-18T18:27:31,362 INFO [AsyncFSWAL-0-hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/MasterData-prefix:961a81da9827,33089,1731954257001 {}] wal.AbstractFSWAL(1368): Slow sync cost: 126 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41713,DS-c75b1747-362b-431f-b47d-543a4f882cdb,DISK], DatanodeInfoWithStorage[127.0.0.1:44191,DS-39ff43bf-c18b-4b86-b37c-eca99a433383,DISK], DatanodeInfoWithStorage[127.0.0.1:34047,DS-1029886b-6b51-4ae4-81a3-156afc7de901,DISK]] 2024-11-18T18:27:31,363 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=43138a6a12c2efea7093064622bb3cd8, ASSIGN because future has completed 2024-11-18T18:27:31,364 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=81f0db1bea1e11879dc1cafb06c0756d, ASSIGN because future has completed 2024-11-18T18:27:31,365 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 43138a6a12c2efea7093064622bb3cd8, server=961a81da9827,42895,1731954257867}] 2024-11-18T18:27:31,366 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5e69ef2147d87d1b83dbcbb4705cbabe, ASSIGN because future has completed 2024-11-18T18:27:31,367 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c3e2f3fb6516fcaaef4d0a2c8b63842d, ASSIGN because future has completed 2024-11-18T18:27:31,368 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=bcf8695060a326b7d24615ba6cb932b1, ASSIGN because future has completed 2024-11-18T18:27:31,370 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure bcf8695060a326b7d24615ba6cb932b1, server=961a81da9827,38709,1731954257723}] 2024-11-18T18:27:31,396 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure c3e2f3fb6516fcaaef4d0a2c8b63842d, server=961a81da9827,38709,1731954257723}] 2024-11-18T18:27:31,481 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5e69ef2147d87d1b83dbcbb4705cbabe, server=961a81da9827,38709,1731954257723}] 2024-11-18T18:27:31,485 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 81f0db1bea1e11879dc1cafb06c0756d, server=961a81da9827,38709,1731954257723}] 2024-11-18T18:27:31,491 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 40a213dc6b2a0fd809c71612c437ab83, server=961a81da9827,42895,1731954257867}] 2024-11-18T18:27:31,494 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a244d0ab318eb4dd77bc5235828d9d51, server=961a81da9827,42895,1731954257867}] 2024-11-18T18:27:31,501 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731954260238.779798cd60db955ed78a4329cac08969. 2024-11-18T18:27:31,501 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7752): Opening region: {ENCODED => 779798cd60db955ed78a4329cac08969, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731954260238.779798cd60db955ed78a4329cac08969.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-18T18:27:31,502 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 779798cd60db955ed78a4329cac08969 2024-11-18T18:27:31,502 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731954260238.779798cd60db955ed78a4329cac08969.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:27:31,502 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7794): checking encryption for 779798cd60db955ed78a4329cac08969 2024-11-18T18:27:31,502 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7797): checking classloading for 779798cd60db955ed78a4329cac08969 2024-11-18T18:27:31,503 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1ad74f79ba4d7fd1619c83a55106331, server=961a81da9827,42895,1731954257867}] 2024-11-18T18:27:31,515 INFO [StoreOpener-779798cd60db955ed78a4329cac08969-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 779798cd60db955ed78a4329cac08969 2024-11-18T18:27:31,516 INFO [StoreOpener-779798cd60db955ed78a4329cac08969-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 779798cd60db955ed78a4329cac08969 columnFamilyName colfamily11 2024-11-18T18:27:31,516 DEBUG [StoreOpener-779798cd60db955ed78a4329cac08969-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:27:31,517 INFO [StoreOpener-779798cd60db955ed78a4329cac08969-1 {}] regionserver.HStore(327): Store=779798cd60db955ed78a4329cac08969/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:27:31,517 INFO [StoreOpener-779798cd60db955ed78a4329cac08969-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 779798cd60db955ed78a4329cac08969 2024-11-18T18:27:31,518 INFO [StoreOpener-779798cd60db955ed78a4329cac08969-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 779798cd60db955ed78a4329cac08969 columnFamilyName colfamily21 2024-11-18T18:27:31,518 DEBUG [StoreOpener-779798cd60db955ed78a4329cac08969-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:27:31,518 INFO [StoreOpener-779798cd60db955ed78a4329cac08969-1 {}] regionserver.HStore(327): Store=779798cd60db955ed78a4329cac08969/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:27:31,519 INFO [StoreOpener-779798cd60db955ed78a4329cac08969-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 779798cd60db955ed78a4329cac08969 2024-11-18T18:27:31,519 INFO [StoreOpener-779798cd60db955ed78a4329cac08969-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 779798cd60db955ed78a4329cac08969 columnFamilyName colfamily31 2024-11-18T18:27:31,519 DEBUG [StoreOpener-779798cd60db955ed78a4329cac08969-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:27:31,520 INFO [StoreOpener-779798cd60db955ed78a4329cac08969-1 {}] regionserver.HStore(327): Store=779798cd60db955ed78a4329cac08969/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:27:31,520 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1038): replaying wal for 779798cd60db955ed78a4329cac08969 2024-11-18T18:27:31,521 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/779798cd60db955ed78a4329cac08969 2024-11-18T18:27:31,524 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/779798cd60db955ed78a4329cac08969 2024-11-18T18:27:31,525 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/779798cd60db955ed78a4329cac08969/recovered.edits/0000000000000000004 2024-11-18T18:27:31,533 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/779798cd60db955ed78a4329cac08969/recovered.edits/0000000000000000004 2024-11-18T18:27:31,533 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(2902): Flushing 779798cd60db955ed78a4329cac08969 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-18T18:27:31,538 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44409, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-18T18:27:31,560 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731954260238.5e69ef2147d87d1b83dbcbb4705cbabe. 2024-11-18T18:27:31,561 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => 5e69ef2147d87d1b83dbcbb4705cbabe, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731954260238.5e69ef2147d87d1b83dbcbb4705cbabe.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-18T18:27:31,561 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 5e69ef2147d87d1b83dbcbb4705cbabe 2024-11-18T18:27:31,561 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731954260238.5e69ef2147d87d1b83dbcbb4705cbabe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:27:31,561 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for 5e69ef2147d87d1b83dbcbb4705cbabe 2024-11-18T18:27:31,561 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for 5e69ef2147d87d1b83dbcbb4705cbabe 2024-11-18T18:27:31,580 INFO [StoreOpener-5e69ef2147d87d1b83dbcbb4705cbabe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 5e69ef2147d87d1b83dbcbb4705cbabe 2024-11-18T18:27:31,581 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T18:27:31,581 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/779798cd60db955ed78a4329cac08969/.tmp/colfamily11/dea3c5fdd90a4f40b9850f7ba9464640 is 52, key is yyy/colfamily11:colfamily11/1731954264741/Put/seqid=0 2024-11-18T18:27:31,583 INFO [StoreOpener-5e69ef2147d87d1b83dbcbb4705cbabe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5e69ef2147d87d1b83dbcbb4705cbabe columnFamilyName colfamily11 2024-11-18T18:27:31,583 DEBUG [StoreOpener-5e69ef2147d87d1b83dbcbb4705cbabe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:27:31,584 INFO [StoreOpener-5e69ef2147d87d1b83dbcbb4705cbabe-1 {}] regionserver.HStore(327): Store=5e69ef2147d87d1b83dbcbb4705cbabe/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:27:31,584 INFO [StoreOpener-5e69ef2147d87d1b83dbcbb4705cbabe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 5e69ef2147d87d1b83dbcbb4705cbabe 2024-11-18T18:27:31,594 INFO [StoreOpener-5e69ef2147d87d1b83dbcbb4705cbabe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5e69ef2147d87d1b83dbcbb4705cbabe columnFamilyName colfamily21 2024-11-18T18:27:31,594 DEBUG [StoreOpener-5e69ef2147d87d1b83dbcbb4705cbabe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:27:31,595 INFO [StoreOpener-5e69ef2147d87d1b83dbcbb4705cbabe-1 {}] regionserver.HStore(327): Store=5e69ef2147d87d1b83dbcbb4705cbabe/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:27:31,595 INFO [StoreOpener-5e69ef2147d87d1b83dbcbb4705cbabe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 5e69ef2147d87d1b83dbcbb4705cbabe 2024-11-18T18:27:31,597 INFO [StoreOpener-5e69ef2147d87d1b83dbcbb4705cbabe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5e69ef2147d87d1b83dbcbb4705cbabe columnFamilyName colfamily31 2024-11-18T18:27:31,597 DEBUG [StoreOpener-5e69ef2147d87d1b83dbcbb4705cbabe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:27:31,603 INFO [StoreOpener-5e69ef2147d87d1b83dbcbb4705cbabe-1 {}] regionserver.HStore(327): Store=5e69ef2147d87d1b83dbcbb4705cbabe/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:27:31,603 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for 5e69ef2147d87d1b83dbcbb4705cbabe 2024-11-18T18:27:31,618 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5e69ef2147d87d1b83dbcbb4705cbabe 2024-11-18T18:27:31,634 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5e69ef2147d87d1b83dbcbb4705cbabe 2024-11-18T18:27:31,642 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5e69ef2147d87d1b83dbcbb4705cbabe/recovered.edits/0000000000000000004 2024-11-18T18:27:31,660 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5e69ef2147d87d1b83dbcbb4705cbabe/recovered.edits/0000000000000000004 2024-11-18T18:27:31,660 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(2902): Flushing 5e69ef2147d87d1b83dbcbb4705cbabe 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-18T18:27:31,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741878_1054 (size=39263) 2024-11-18T18:27:31,713 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/779798cd60db955ed78a4329cac08969/.tmp/colfamily11/dea3c5fdd90a4f40b9850f7ba9464640 2024-11-18T18:27:31,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741878_1054 (size=39263) 2024-11-18T18:27:31,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741878_1054 (size=39263) 2024-11-18T18:27:31,727 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/779798cd60db955ed78a4329cac08969/.tmp/colfamily11/dea3c5fdd90a4f40b9850f7ba9464640 as hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/779798cd60db955ed78a4329cac08969/colfamily11/dea3c5fdd90a4f40b9850f7ba9464640 2024-11-18T18:27:31,733 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5e69ef2147d87d1b83dbcbb4705cbabe/.tmp/colfamily11/5f44dd4b778a4bd581bd09aab922f073 is 52, key is ppp/colfamily11:colfamily11/1731954264588/Put/seqid=0 2024-11-18T18:27:31,736 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/779798cd60db955ed78a4329cac08969/colfamily11/dea3c5fdd90a4f40b9850f7ba9464640, entries=703, sequenceid=4, filesize=38.3 K 2024-11-18T18:27:31,736 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 779798cd60db955ed78a4329cac08969 in 203ms, sequenceid=4, compaction requested=false; wal=null 2024-11-18T18:27:31,737 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/779798cd60db955ed78a4329cac08969/recovered.edits/0000000000000000004 2024-11-18T18:27:31,739 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1048): stopping wal replay for 779798cd60db955ed78a4329cac08969 2024-11-18T18:27:31,739 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1060): Cleaning up temporary data for 779798cd60db955ed78a4329cac08969 2024-11-18T18:27:31,740 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T18:27:31,745 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1093): writing seq id for 779798cd60db955ed78a4329cac08969 2024-11-18T18:27:31,753 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/779798cd60db955ed78a4329cac08969/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-18T18:27:31,755 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1114): Opened 779798cd60db955ed78a4329cac08969; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63622946, jitterRate=-0.051944226026535034}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T18:27:31,755 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 779798cd60db955ed78a4329cac08969 2024-11-18T18:27:31,755 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1006): Region open journal for 779798cd60db955ed78a4329cac08969: Running coprocessor pre-open hook at 1731954451502Writing region info on filesystem at 1731954451502Initializing all the Stores at 1731954451514 (+12 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954451514Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954451515 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954451515Obtaining lock to block concurrent updates at 1731954451533 (+18 ms)Preparing flush snapshotting stores in 779798cd60db955ed78a4329cac08969 at 1731954451533Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731954260238.779798cd60db955ed78a4329cac08969., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731954451533Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731954260238.779798cd60db955ed78a4329cac08969. at 1731954451533Flushing 779798cd60db955ed78a4329cac08969/colfamily11: creating writer at 1731954451533Flushing 779798cd60db955ed78a4329cac08969/colfamily11: appending metadata at 1731954451580 (+47 ms)Flushing 779798cd60db955ed78a4329cac08969/colfamily11: closing flushed file at 1731954451581 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4853c68d: reopening flushed file at 1731954451725 (+144 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 779798cd60db955ed78a4329cac08969 in 203ms, sequenceid=4, compaction requested=false; wal=null at 1731954451737 (+12 ms)Cleaning up temporary data from old regions at 1731954451739 (+2 ms)Running coprocessor post-open hooks at 1731954451755 (+16 ms)Region opened successfully at 1731954451755 2024-11-18T18:27:31,757 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731954260238.779798cd60db955ed78a4329cac08969., pid=73, masterSystemTime=1731954451479 2024-11-18T18:27:31,760 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731954260238.779798cd60db955ed78a4329cac08969. 2024-11-18T18:27:31,760 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731954260238.779798cd60db955ed78a4329cac08969. 2024-11-18T18:27:31,760 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731954260238.43138a6a12c2efea7093064622bb3cd8. 2024-11-18T18:27:31,761 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7752): Opening region: {ENCODED => 43138a6a12c2efea7093064622bb3cd8, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731954260238.43138a6a12c2efea7093064622bb3cd8.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-18T18:27:31,761 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 43138a6a12c2efea7093064622bb3cd8 2024-11-18T18:27:31,761 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731954260238.43138a6a12c2efea7093064622bb3cd8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:27:31,761 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=779798cd60db955ed78a4329cac08969, regionState=OPEN, openSeqNum=5, regionLocation=961a81da9827,42895,1731954257867 2024-11-18T18:27:31,761 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7794): checking encryption for 43138a6a12c2efea7093064622bb3cd8 2024-11-18T18:27:31,762 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7797): checking classloading for 43138a6a12c2efea7093064622bb3cd8 2024-11-18T18:27:31,767 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=73, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 779798cd60db955ed78a4329cac08969, server=961a81da9827,42895,1731954257867 because future has completed 2024-11-18T18:27:31,773 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-18T18:27:31,773 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; OpenRegionProcedure 779798cd60db955ed78a4329cac08969, server=961a81da9827,42895,1731954257867 in 537 msec 2024-11-18T18:27:31,775 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=779798cd60db955ed78a4329cac08969, ASSIGN in 713 msec 2024-11-18T18:27:31,775 INFO [StoreOpener-43138a6a12c2efea7093064622bb3cd8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 43138a6a12c2efea7093064622bb3cd8 2024-11-18T18:27:31,778 INFO [StoreOpener-43138a6a12c2efea7093064622bb3cd8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 43138a6a12c2efea7093064622bb3cd8 columnFamilyName colfamily11 2024-11-18T18:27:31,778 DEBUG [StoreOpener-43138a6a12c2efea7093064622bb3cd8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:27:31,780 INFO [StoreOpener-43138a6a12c2efea7093064622bb3cd8-1 {}] regionserver.HStore(327): Store=43138a6a12c2efea7093064622bb3cd8/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:27:31,780 INFO [StoreOpener-43138a6a12c2efea7093064622bb3cd8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 43138a6a12c2efea7093064622bb3cd8 2024-11-18T18:27:31,781 INFO [StoreOpener-43138a6a12c2efea7093064622bb3cd8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 43138a6a12c2efea7093064622bb3cd8 columnFamilyName colfamily21 2024-11-18T18:27:31,781 DEBUG [StoreOpener-43138a6a12c2efea7093064622bb3cd8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:27:31,782 INFO [StoreOpener-43138a6a12c2efea7093064622bb3cd8-1 {}] regionserver.HStore(327): Store=43138a6a12c2efea7093064622bb3cd8/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:27:31,782 INFO [StoreOpener-43138a6a12c2efea7093064622bb3cd8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 43138a6a12c2efea7093064622bb3cd8 2024-11-18T18:27:31,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741879_1055 (size=39263) 2024-11-18T18:27:31,784 INFO [StoreOpener-43138a6a12c2efea7093064622bb3cd8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 43138a6a12c2efea7093064622bb3cd8 columnFamilyName colfamily31 2024-11-18T18:27:31,784 DEBUG [StoreOpener-43138a6a12c2efea7093064622bb3cd8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:27:31,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741879_1055 (size=39263) 2024-11-18T18:27:31,785 INFO [StoreOpener-43138a6a12c2efea7093064622bb3cd8-1 {}] regionserver.HStore(327): Store=43138a6a12c2efea7093064622bb3cd8/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:27:31,786 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1038): replaying wal for 43138a6a12c2efea7093064622bb3cd8 2024-11-18T18:27:31,787 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5e69ef2147d87d1b83dbcbb4705cbabe/.tmp/colfamily11/5f44dd4b778a4bd581bd09aab922f073 2024-11-18T18:27:31,790 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/43138a6a12c2efea7093064622bb3cd8 2024-11-18T18:27:31,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741879_1055 (size=39263) 2024-11-18T18:27:31,798 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5e69ef2147d87d1b83dbcbb4705cbabe/.tmp/colfamily11/5f44dd4b778a4bd581bd09aab922f073 as hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5e69ef2147d87d1b83dbcbb4705cbabe/colfamily11/5f44dd4b778a4bd581bd09aab922f073 2024-11-18T18:27:31,802 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/43138a6a12c2efea7093064622bb3cd8 2024-11-18T18:27:31,807 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5e69ef2147d87d1b83dbcbb4705cbabe/colfamily11/5f44dd4b778a4bd581bd09aab922f073, entries=703, sequenceid=4, filesize=38.3 K 2024-11-18T18:27:31,808 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 5e69ef2147d87d1b83dbcbb4705cbabe in 148ms, sequenceid=4, compaction requested=false; wal=null 2024-11-18T18:27:31,809 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5e69ef2147d87d1b83dbcbb4705cbabe/recovered.edits/0000000000000000004 2024-11-18T18:27:31,812 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/43138a6a12c2efea7093064622bb3cd8/recovered.edits/0000000000000000004 2024-11-18T18:27:31,817 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for 5e69ef2147d87d1b83dbcbb4705cbabe 2024-11-18T18:27:31,817 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for 5e69ef2147d87d1b83dbcbb4705cbabe 2024-11-18T18:27:31,818 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T18:27:31,821 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for 5e69ef2147d87d1b83dbcbb4705cbabe 2024-11-18T18:27:31,824 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/43138a6a12c2efea7093064622bb3cd8/recovered.edits/0000000000000000004 2024-11-18T18:27:31,824 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(2902): Flushing 43138a6a12c2efea7093064622bb3cd8 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-18T18:27:31,829 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5e69ef2147d87d1b83dbcbb4705cbabe/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-18T18:27:31,830 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened 5e69ef2147d87d1b83dbcbb4705cbabe; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64212763, jitterRate=-0.04315526783466339}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T18:27:31,830 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5e69ef2147d87d1b83dbcbb4705cbabe 2024-11-18T18:27:31,830 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for 5e69ef2147d87d1b83dbcbb4705cbabe: Running coprocessor pre-open hook at 1731954451562Writing region info on filesystem at 1731954451562Initializing all the Stores at 1731954451569 (+7 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954451569Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954451577 (+8 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954451577Obtaining lock to block concurrent updates at 1731954451660 (+83 ms)Preparing flush snapshotting stores in 5e69ef2147d87d1b83dbcbb4705cbabe at 1731954451660Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731954260238.5e69ef2147d87d1b83dbcbb4705cbabe., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731954451660Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731954260238.5e69ef2147d87d1b83dbcbb4705cbabe. at 1731954451660Flushing 5e69ef2147d87d1b83dbcbb4705cbabe/colfamily11: creating writer at 1731954451660Flushing 5e69ef2147d87d1b83dbcbb4705cbabe/colfamily11: appending metadata at 1731954451733 (+73 ms)Flushing 5e69ef2147d87d1b83dbcbb4705cbabe/colfamily11: closing flushed file at 1731954451733Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1dc014b0: reopening flushed file at 1731954451796 (+63 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 5e69ef2147d87d1b83dbcbb4705cbabe in 148ms, sequenceid=4, compaction requested=false; wal=null at 1731954451808 (+12 ms)Cleaning up temporary data from old regions at 1731954451817 (+9 ms)Running coprocessor post-open hooks at 1731954451830 (+13 ms)Region opened successfully at 1731954451830 2024-11-18T18:27:31,831 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731954260238.5e69ef2147d87d1b83dbcbb4705cbabe., pid=77, masterSystemTime=1731954451536 2024-11-18T18:27:31,835 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731954260238.5e69ef2147d87d1b83dbcbb4705cbabe. 2024-11-18T18:27:31,835 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731954260238.5e69ef2147d87d1b83dbcbb4705cbabe. 2024-11-18T18:27:31,835 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731954260238.bcf8695060a326b7d24615ba6cb932b1. 2024-11-18T18:27:31,837 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => bcf8695060a326b7d24615ba6cb932b1, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731954260238.bcf8695060a326b7d24615ba6cb932b1.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-18T18:27:31,837 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true bcf8695060a326b7d24615ba6cb932b1 2024-11-18T18:27:31,838 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=5e69ef2147d87d1b83dbcbb4705cbabe, regionState=OPEN, openSeqNum=5, regionLocation=961a81da9827,38709,1731954257723 2024-11-18T18:27:31,837 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731954260238.bcf8695060a326b7d24615ba6cb932b1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:27:31,839 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for bcf8695060a326b7d24615ba6cb932b1 2024-11-18T18:27:31,839 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for bcf8695060a326b7d24615ba6cb932b1 2024-11-18T18:27:31,842 INFO [StoreOpener-bcf8695060a326b7d24615ba6cb932b1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region bcf8695060a326b7d24615ba6cb932b1 2024-11-18T18:27:31,842 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=77, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5e69ef2147d87d1b83dbcbb4705cbabe, server=961a81da9827,38709,1731954257723 because future has completed 2024-11-18T18:27:31,846 INFO [StoreOpener-bcf8695060a326b7d24615ba6cb932b1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bcf8695060a326b7d24615ba6cb932b1 columnFamilyName colfamily11 2024-11-18T18:27:31,847 DEBUG [StoreOpener-bcf8695060a326b7d24615ba6cb932b1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:27:31,847 INFO [StoreOpener-bcf8695060a326b7d24615ba6cb932b1-1 {}] regionserver.HStore(327): Store=bcf8695060a326b7d24615ba6cb932b1/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:27:31,847 INFO [StoreOpener-bcf8695060a326b7d24615ba6cb932b1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region bcf8695060a326b7d24615ba6cb932b1 2024-11-18T18:27:31,848 INFO [StoreOpener-bcf8695060a326b7d24615ba6cb932b1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bcf8695060a326b7d24615ba6cb932b1 columnFamilyName colfamily21 2024-11-18T18:27:31,849 DEBUG [StoreOpener-bcf8695060a326b7d24615ba6cb932b1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:27:31,850 INFO [StoreOpener-bcf8695060a326b7d24615ba6cb932b1-1 {}] regionserver.HStore(327): Store=bcf8695060a326b7d24615ba6cb932b1/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:27:31,850 INFO [StoreOpener-bcf8695060a326b7d24615ba6cb932b1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region bcf8695060a326b7d24615ba6cb932b1 2024-11-18T18:27:31,851 INFO [StoreOpener-bcf8695060a326b7d24615ba6cb932b1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bcf8695060a326b7d24615ba6cb932b1 columnFamilyName colfamily31 2024-11-18T18:27:31,851 DEBUG [StoreOpener-bcf8695060a326b7d24615ba6cb932b1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:27:31,858 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=64 2024-11-18T18:27:31,858 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure 5e69ef2147d87d1b83dbcbb4705cbabe, server=961a81da9827,38709,1731954257723 in 368 msec 2024-11-18T18:27:31,861 INFO [StoreOpener-bcf8695060a326b7d24615ba6cb932b1-1 {}] regionserver.HStore(327): Store=bcf8695060a326b7d24615ba6cb932b1/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:27:31,862 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for bcf8695060a326b7d24615ba6cb932b1 2024-11-18T18:27:31,863 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bcf8695060a326b7d24615ba6cb932b1 2024-11-18T18:27:31,865 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bcf8695060a326b7d24615ba6cb932b1 2024-11-18T18:27:31,866 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bcf8695060a326b7d24615ba6cb932b1/recovered.edits/0000000000000000004 2024-11-18T18:27:31,870 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5e69ef2147d87d1b83dbcbb4705cbabe, ASSIGN in 798 msec 2024-11-18T18:27:31,874 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bcf8695060a326b7d24615ba6cb932b1/recovered.edits/0000000000000000004 2024-11-18T18:27:31,875 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(2902): Flushing bcf8695060a326b7d24615ba6cb932b1 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-18T18:27:31,875 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/43138a6a12c2efea7093064622bb3cd8/.tmp/colfamily11/e5095bfae61c4ce7a15cf5f23d01a4b5 is 52, key is ccc/colfamily11:colfamily11/1731954264221/Put/seqid=0 2024-11-18T18:27:31,920 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bcf8695060a326b7d24615ba6cb932b1/.tmp/colfamily11/52887ba7b81648f4b1630e2ae0f6e3ad is 52, key is eee/colfamily11:colfamily11/1731954264300/Put/seqid=0 2024-11-18T18:27:31,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741881_1057 (size=39263) 2024-11-18T18:27:31,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741880_1056 (size=39263) 2024-11-18T18:27:31,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741880_1056 (size=39263) 2024-11-18T18:27:31,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741881_1057 (size=39263) 2024-11-18T18:27:31,946 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bcf8695060a326b7d24615ba6cb932b1/.tmp/colfamily11/52887ba7b81648f4b1630e2ae0f6e3ad 2024-11-18T18:27:31,946 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/43138a6a12c2efea7093064622bb3cd8/.tmp/colfamily11/e5095bfae61c4ce7a15cf5f23d01a4b5 2024-11-18T18:27:31,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741881_1057 (size=39263) 2024-11-18T18:27:31,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741880_1056 (size=39263) 2024-11-18T18:27:31,969 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bcf8695060a326b7d24615ba6cb932b1/.tmp/colfamily11/52887ba7b81648f4b1630e2ae0f6e3ad as hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bcf8695060a326b7d24615ba6cb932b1/colfamily11/52887ba7b81648f4b1630e2ae0f6e3ad 2024-11-18T18:27:31,972 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/43138a6a12c2efea7093064622bb3cd8/.tmp/colfamily11/e5095bfae61c4ce7a15cf5f23d01a4b5 as hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/43138a6a12c2efea7093064622bb3cd8/colfamily11/e5095bfae61c4ce7a15cf5f23d01a4b5 2024-11-18T18:27:31,983 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/43138a6a12c2efea7093064622bb3cd8/colfamily11/e5095bfae61c4ce7a15cf5f23d01a4b5, entries=703, sequenceid=4, filesize=38.3 K 2024-11-18T18:27:31,983 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 43138a6a12c2efea7093064622bb3cd8 in 159ms, sequenceid=4, compaction requested=false; wal=null 2024-11-18T18:27:31,986 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/43138a6a12c2efea7093064622bb3cd8/recovered.edits/0000000000000000004 2024-11-18T18:27:31,990 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bcf8695060a326b7d24615ba6cb932b1/colfamily11/52887ba7b81648f4b1630e2ae0f6e3ad, entries=703, sequenceid=4, filesize=38.3 K 2024-11-18T18:27:31,991 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for bcf8695060a326b7d24615ba6cb932b1 in 116ms, sequenceid=4, compaction requested=false; wal=null 2024-11-18T18:27:31,992 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1048): stopping wal replay for 43138a6a12c2efea7093064622bb3cd8 2024-11-18T18:27:31,992 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1060): Cleaning up temporary data for 43138a6a12c2efea7093064622bb3cd8 2024-11-18T18:27:31,993 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T18:27:31,994 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bcf8695060a326b7d24615ba6cb932b1/recovered.edits/0000000000000000004 2024-11-18T18:27:31,998 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1093): writing seq id for 43138a6a12c2efea7093064622bb3cd8 2024-11-18T18:27:32,005 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/43138a6a12c2efea7093064622bb3cd8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-18T18:27:32,006 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for bcf8695060a326b7d24615ba6cb932b1 2024-11-18T18:27:32,006 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for bcf8695060a326b7d24615ba6cb932b1 2024-11-18T18:27:32,008 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1114): Opened 43138a6a12c2efea7093064622bb3cd8; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60198370, jitterRate=-0.10297438502311707}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T18:27:32,008 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 43138a6a12c2efea7093064622bb3cd8 2024-11-18T18:27:32,008 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1006): Region open journal for 43138a6a12c2efea7093064622bb3cd8: Running coprocessor pre-open hook at 1731954451762Writing region info on filesystem at 1731954451762Initializing all the Stores at 1731954451765 (+3 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954451765Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954451775 (+10 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954451775Obtaining lock to block concurrent updates at 1731954451824 (+49 ms)Preparing flush snapshotting stores in 43138a6a12c2efea7093064622bb3cd8 at 1731954451824Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731954260238.43138a6a12c2efea7093064622bb3cd8., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731954451825 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731954260238.43138a6a12c2efea7093064622bb3cd8. at 1731954451825Flushing 43138a6a12c2efea7093064622bb3cd8/colfamily11: creating writer at 1731954451825Flushing 43138a6a12c2efea7093064622bb3cd8/colfamily11: appending metadata at 1731954451874 (+49 ms)Flushing 43138a6a12c2efea7093064622bb3cd8/colfamily11: closing flushed file at 1731954451874Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@617a403c: reopening flushed file at 1731954451965 (+91 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 43138a6a12c2efea7093064622bb3cd8 in 159ms, sequenceid=4, compaction requested=false; wal=null at 1731954451983 (+18 ms)Cleaning up temporary data from old regions at 1731954451992 (+9 ms)Running coprocessor post-open hooks at 1731954452008 (+16 ms)Region opened successfully at 1731954452008 2024-11-18T18:27:32,010 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T18:27:32,018 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731954260238.43138a6a12c2efea7093064622bb3cd8., pid=74, masterSystemTime=1731954451479 2024-11-18T18:27:32,018 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for bcf8695060a326b7d24615ba6cb932b1 2024-11-18T18:27:32,022 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731954260238.43138a6a12c2efea7093064622bb3cd8. 2024-11-18T18:27:32,022 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731954260238.43138a6a12c2efea7093064622bb3cd8. 2024-11-18T18:27:32,023 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731954260238.a244d0ab318eb4dd77bc5235828d9d51. 2024-11-18T18:27:32,023 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7752): Opening region: {ENCODED => a244d0ab318eb4dd77bc5235828d9d51, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731954260238.a244d0ab318eb4dd77bc5235828d9d51.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-18T18:27:32,023 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true a244d0ab318eb4dd77bc5235828d9d51 2024-11-18T18:27:32,024 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731954260238.a244d0ab318eb4dd77bc5235828d9d51.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:27:32,024 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7794): checking encryption for a244d0ab318eb4dd77bc5235828d9d51 2024-11-18T18:27:32,024 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7797): checking classloading for a244d0ab318eb4dd77bc5235828d9d51 2024-11-18T18:27:32,026 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=43138a6a12c2efea7093064622bb3cd8, regionState=OPEN, openSeqNum=5, regionLocation=961a81da9827,42895,1731954257867 2024-11-18T18:27:32,028 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bcf8695060a326b7d24615ba6cb932b1/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-18T18:27:32,030 INFO [StoreOpener-a244d0ab318eb4dd77bc5235828d9d51-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region a244d0ab318eb4dd77bc5235828d9d51 2024-11-18T18:27:32,031 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened bcf8695060a326b7d24615ba6cb932b1; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65499496, jitterRate=-0.023981451988220215}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T18:27:32,031 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bcf8695060a326b7d24615ba6cb932b1 2024-11-18T18:27:32,031 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for bcf8695060a326b7d24615ba6cb932b1: Running coprocessor pre-open hook at 1731954451839Writing region info on filesystem at 1731954451839Initializing all the Stores at 1731954451840 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954451840Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954451842 (+2 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954451842Obtaining lock to block concurrent updates at 1731954451875 (+33 ms)Preparing flush snapshotting stores in bcf8695060a326b7d24615ba6cb932b1 at 1731954451875Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731954260238.bcf8695060a326b7d24615ba6cb932b1., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731954451875Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731954260238.bcf8695060a326b7d24615ba6cb932b1. at 1731954451875Flushing bcf8695060a326b7d24615ba6cb932b1/colfamily11: creating writer at 1731954451875Flushing bcf8695060a326b7d24615ba6cb932b1/colfamily11: appending metadata at 1731954451919 (+44 ms)Flushing bcf8695060a326b7d24615ba6cb932b1/colfamily11: closing flushed file at 1731954451919Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@20c3ee44: reopening flushed file at 1731954451964 (+45 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for bcf8695060a326b7d24615ba6cb932b1 in 116ms, sequenceid=4, compaction requested=false; wal=null at 1731954451991 (+27 ms)Cleaning up temporary data from old regions at 1731954452006 (+15 ms)Running coprocessor post-open hooks at 1731954452031 (+25 ms)Region opened successfully at 1731954452031 2024-11-18T18:27:32,031 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=74, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 43138a6a12c2efea7093064622bb3cd8, server=961a81da9827,42895,1731954257867 because future has completed 2024-11-18T18:27:32,032 INFO [StoreOpener-a244d0ab318eb4dd77bc5235828d9d51-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a244d0ab318eb4dd77bc5235828d9d51 columnFamilyName colfamily11 2024-11-18T18:27:32,032 DEBUG [StoreOpener-a244d0ab318eb4dd77bc5235828d9d51-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:27:32,037 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731954260238.bcf8695060a326b7d24615ba6cb932b1., pid=75, masterSystemTime=1731954451536 2024-11-18T18:27:32,033 INFO [StoreOpener-a244d0ab318eb4dd77bc5235828d9d51-1 {}] regionserver.HStore(327): Store=a244d0ab318eb4dd77bc5235828d9d51/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:27:32,040 INFO [StoreOpener-a244d0ab318eb4dd77bc5235828d9d51-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region a244d0ab318eb4dd77bc5235828d9d51 2024-11-18T18:27:32,043 INFO [StoreOpener-a244d0ab318eb4dd77bc5235828d9d51-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a244d0ab318eb4dd77bc5235828d9d51 columnFamilyName colfamily21 2024-11-18T18:27:32,043 DEBUG [StoreOpener-a244d0ab318eb4dd77bc5235828d9d51-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:27:32,046 INFO [StoreOpener-a244d0ab318eb4dd77bc5235828d9d51-1 {}] regionserver.HStore(327): Store=a244d0ab318eb4dd77bc5235828d9d51/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:27:32,047 INFO [StoreOpener-a244d0ab318eb4dd77bc5235828d9d51-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region a244d0ab318eb4dd77bc5235828d9d51 2024-11-18T18:27:32,048 INFO [StoreOpener-a244d0ab318eb4dd77bc5235828d9d51-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a244d0ab318eb4dd77bc5235828d9d51 columnFamilyName colfamily31 2024-11-18T18:27:32,048 DEBUG [StoreOpener-a244d0ab318eb4dd77bc5235828d9d51-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:27:32,049 INFO [StoreOpener-a244d0ab318eb4dd77bc5235828d9d51-1 {}] regionserver.HStore(327): Store=a244d0ab318eb4dd77bc5235828d9d51/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:27:32,049 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1038): replaying wal for a244d0ab318eb4dd77bc5235828d9d51 2024-11-18T18:27:32,050 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731954260238.bcf8695060a326b7d24615ba6cb932b1. 2024-11-18T18:27:32,050 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731954260238.bcf8695060a326b7d24615ba6cb932b1. 2024-11-18T18:27:32,050 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731954260238.c3e2f3fb6516fcaaef4d0a2c8b63842d. 2024-11-18T18:27:32,050 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => c3e2f3fb6516fcaaef4d0a2c8b63842d, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731954260238.c3e2f3fb6516fcaaef4d0a2c8b63842d.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-18T18:27:32,051 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true c3e2f3fb6516fcaaef4d0a2c8b63842d 2024-11-18T18:27:32,051 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731954260238.c3e2f3fb6516fcaaef4d0a2c8b63842d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:27:32,051 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for c3e2f3fb6516fcaaef4d0a2c8b63842d 2024-11-18T18:27:32,051 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for c3e2f3fb6516fcaaef4d0a2c8b63842d 2024-11-18T18:27:32,057 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a244d0ab318eb4dd77bc5235828d9d51 2024-11-18T18:27:32,058 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=74, resume processing ppid=66 2024-11-18T18:27:32,059 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, ppid=66, state=SUCCESS, hasLock=false; OpenRegionProcedure 43138a6a12c2efea7093064622bb3cd8, server=961a81da9827,42895,1731954257867 in 672 msec 2024-11-18T18:27:32,059 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a244d0ab318eb4dd77bc5235828d9d51 2024-11-18T18:27:32,060 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a244d0ab318eb4dd77bc5235828d9d51/recovered.edits/0000000000000000004 2024-11-18T18:27:32,068 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a244d0ab318eb4dd77bc5235828d9d51/recovered.edits/0000000000000000004 2024-11-18T18:27:32,068 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(2902): Flushing a244d0ab318eb4dd77bc5235828d9d51 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-18T18:27:32,073 INFO [StoreOpener-c3e2f3fb6516fcaaef4d0a2c8b63842d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region c3e2f3fb6516fcaaef4d0a2c8b63842d 2024-11-18T18:27:32,074 INFO [StoreOpener-c3e2f3fb6516fcaaef4d0a2c8b63842d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c3e2f3fb6516fcaaef4d0a2c8b63842d columnFamilyName colfamily11 2024-11-18T18:27:32,075 DEBUG [StoreOpener-c3e2f3fb6516fcaaef4d0a2c8b63842d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:27:32,075 INFO [StoreOpener-c3e2f3fb6516fcaaef4d0a2c8b63842d-1 {}] regionserver.HStore(327): Store=c3e2f3fb6516fcaaef4d0a2c8b63842d/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:27:32,075 INFO [StoreOpener-c3e2f3fb6516fcaaef4d0a2c8b63842d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region c3e2f3fb6516fcaaef4d0a2c8b63842d 2024-11-18T18:27:32,078 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=43138a6a12c2efea7093064622bb3cd8, ASSIGN in 999 msec 2024-11-18T18:27:32,079 INFO [StoreOpener-c3e2f3fb6516fcaaef4d0a2c8b63842d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c3e2f3fb6516fcaaef4d0a2c8b63842d columnFamilyName colfamily21 2024-11-18T18:27:32,079 DEBUG [StoreOpener-c3e2f3fb6516fcaaef4d0a2c8b63842d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:27:32,079 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=65 updating hbase:meta row=bcf8695060a326b7d24615ba6cb932b1, regionState=OPEN, openSeqNum=5, regionLocation=961a81da9827,38709,1731954257723 2024-11-18T18:27:32,082 INFO [StoreOpener-c3e2f3fb6516fcaaef4d0a2c8b63842d-1 {}] regionserver.HStore(327): Store=c3e2f3fb6516fcaaef4d0a2c8b63842d/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:27:32,082 INFO [StoreOpener-c3e2f3fb6516fcaaef4d0a2c8b63842d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region c3e2f3fb6516fcaaef4d0a2c8b63842d 2024-11-18T18:27:32,083 INFO [StoreOpener-c3e2f3fb6516fcaaef4d0a2c8b63842d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c3e2f3fb6516fcaaef4d0a2c8b63842d columnFamilyName colfamily31 2024-11-18T18:27:32,083 DEBUG [StoreOpener-c3e2f3fb6516fcaaef4d0a2c8b63842d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:27:32,084 INFO [StoreOpener-c3e2f3fb6516fcaaef4d0a2c8b63842d-1 {}] regionserver.HStore(327): Store=c3e2f3fb6516fcaaef4d0a2c8b63842d/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:27:32,084 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for c3e2f3fb6516fcaaef4d0a2c8b63842d 2024-11-18T18:27:32,085 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c3e2f3fb6516fcaaef4d0a2c8b63842d 2024-11-18T18:27:32,087 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c3e2f3fb6516fcaaef4d0a2c8b63842d 2024-11-18T18:27:32,088 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c3e2f3fb6516fcaaef4d0a2c8b63842d/recovered.edits/0000000000000000004 2024-11-18T18:27:32,089 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure bcf8695060a326b7d24615ba6cb932b1, server=961a81da9827,38709,1731954257723 because future has completed 2024-11-18T18:27:32,097 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=65 2024-11-18T18:27:32,097 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=65, state=SUCCESS, hasLock=false; OpenRegionProcedure bcf8695060a326b7d24615ba6cb932b1, server=961a81da9827,38709,1731954257723 in 724 msec 2024-11-18T18:27:32,097 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c3e2f3fb6516fcaaef4d0a2c8b63842d/recovered.edits/0000000000000000004 2024-11-18T18:27:32,098 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(2902): Flushing c3e2f3fb6516fcaaef4d0a2c8b63842d 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-18T18:27:32,108 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=bcf8695060a326b7d24615ba6cb932b1, ASSIGN in 1.0370 sec 2024-11-18T18:27:32,131 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c3e2f3fb6516fcaaef4d0a2c8b63842d/.tmp/colfamily11/b6194ad4e4a1423cafe83f481c1f1ff6 is 52, key is kkk/colfamily11:colfamily11/1731954264489/Put/seqid=0 2024-11-18T18:27:32,132 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a244d0ab318eb4dd77bc5235828d9d51/.tmp/colfamily11/2912ee6b4009405da1ee07b80dab4880 is 52, key is mmm/colfamily11:colfamily11/1731954264544/Put/seqid=0 2024-11-18T18:27:32,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741882_1058 (size=39263) 2024-11-18T18:27:32,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741882_1058 (size=39263) 2024-11-18T18:27:32,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741882_1058 (size=39263) 2024-11-18T18:27:32,169 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c3e2f3fb6516fcaaef4d0a2c8b63842d/.tmp/colfamily11/b6194ad4e4a1423cafe83f481c1f1ff6 2024-11-18T18:27:32,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741883_1059 (size=39263) 2024-11-18T18:27:32,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741883_1059 (size=39263) 2024-11-18T18:27:32,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741883_1059 (size=39263) 2024-11-18T18:27:32,186 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a244d0ab318eb4dd77bc5235828d9d51/.tmp/colfamily11/2912ee6b4009405da1ee07b80dab4880 2024-11-18T18:27:32,209 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c3e2f3fb6516fcaaef4d0a2c8b63842d/.tmp/colfamily11/b6194ad4e4a1423cafe83f481c1f1ff6 as hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c3e2f3fb6516fcaaef4d0a2c8b63842d/colfamily11/b6194ad4e4a1423cafe83f481c1f1ff6 2024-11-18T18:27:32,217 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a244d0ab318eb4dd77bc5235828d9d51/.tmp/colfamily11/2912ee6b4009405da1ee07b80dab4880 as hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a244d0ab318eb4dd77bc5235828d9d51/colfamily11/2912ee6b4009405da1ee07b80dab4880 2024-11-18T18:27:32,227 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c3e2f3fb6516fcaaef4d0a2c8b63842d/colfamily11/b6194ad4e4a1423cafe83f481c1f1ff6, entries=703, sequenceid=4, filesize=38.3 K 2024-11-18T18:27:32,227 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for c3e2f3fb6516fcaaef4d0a2c8b63842d in 130ms, sequenceid=4, compaction requested=false; wal=null 2024-11-18T18:27:32,228 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c3e2f3fb6516fcaaef4d0a2c8b63842d/recovered.edits/0000000000000000004 2024-11-18T18:27:32,232 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for c3e2f3fb6516fcaaef4d0a2c8b63842d 2024-11-18T18:27:32,232 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for c3e2f3fb6516fcaaef4d0a2c8b63842d 2024-11-18T18:27:32,235 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T18:27:32,240 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a244d0ab318eb4dd77bc5235828d9d51/colfamily11/2912ee6b4009405da1ee07b80dab4880, entries=703, sequenceid=4, filesize=38.3 K 2024-11-18T18:27:32,241 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for c3e2f3fb6516fcaaef4d0a2c8b63842d 2024-11-18T18:27:32,241 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for a244d0ab318eb4dd77bc5235828d9d51 in 173ms, sequenceid=4, compaction requested=false; wal=null 2024-11-18T18:27:32,242 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a244d0ab318eb4dd77bc5235828d9d51/recovered.edits/0000000000000000004 2024-11-18T18:27:32,249 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c3e2f3fb6516fcaaef4d0a2c8b63842d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-18T18:27:32,250 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1048): stopping wal replay for a244d0ab318eb4dd77bc5235828d9d51 2024-11-18T18:27:32,251 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1060): Cleaning up temporary data for a244d0ab318eb4dd77bc5235828d9d51 2024-11-18T18:27:32,251 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened c3e2f3fb6516fcaaef4d0a2c8b63842d; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62159693, jitterRate=-0.0737483948469162}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T18:27:32,251 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c3e2f3fb6516fcaaef4d0a2c8b63842d 2024-11-18T18:27:32,251 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for c3e2f3fb6516fcaaef4d0a2c8b63842d: Running coprocessor pre-open hook at 1731954452051Writing region info on filesystem at 1731954452051Initializing all the Stores at 1731954452056 (+5 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954452057 (+1 ms)Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954452065 (+8 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954452065Obtaining lock to block concurrent updates at 1731954452098 (+33 ms)Preparing flush snapshotting stores in c3e2f3fb6516fcaaef4d0a2c8b63842d at 1731954452098Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731954260238.c3e2f3fb6516fcaaef4d0a2c8b63842d., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731954452098Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731954260238.c3e2f3fb6516fcaaef4d0a2c8b63842d. at 1731954452098Flushing c3e2f3fb6516fcaaef4d0a2c8b63842d/colfamily11: creating writer at 1731954452098Flushing c3e2f3fb6516fcaaef4d0a2c8b63842d/colfamily11: appending metadata at 1731954452131 (+33 ms)Flushing c3e2f3fb6516fcaaef4d0a2c8b63842d/colfamily11: closing flushed file at 1731954452131Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6fc64141: reopening flushed file at 1731954452189 (+58 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for c3e2f3fb6516fcaaef4d0a2c8b63842d in 130ms, sequenceid=4, compaction requested=false; wal=null at 1731954452227 (+38 ms)Cleaning up temporary data from old regions at 1731954452232 (+5 ms)Running coprocessor post-open hooks at 1731954452251 (+19 ms)Region opened successfully at 1731954452251 2024-11-18T18:27:32,253 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731954260238.c3e2f3fb6516fcaaef4d0a2c8b63842d., pid=76, masterSystemTime=1731954451536 2024-11-18T18:27:32,253 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T18:27:32,263 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731954260238.c3e2f3fb6516fcaaef4d0a2c8b63842d. 2024-11-18T18:27:32,264 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1093): writing seq id for a244d0ab318eb4dd77bc5235828d9d51 2024-11-18T18:27:32,264 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=c3e2f3fb6516fcaaef4d0a2c8b63842d, regionState=OPEN, openSeqNum=5, regionLocation=961a81da9827,38709,1731954257723 2024-11-18T18:27:32,267 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731954260238.c3e2f3fb6516fcaaef4d0a2c8b63842d. 2024-11-18T18:27:32,267 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731954260238.81f0db1bea1e11879dc1cafb06c0756d. 2024-11-18T18:27:32,267 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => 81f0db1bea1e11879dc1cafb06c0756d, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731954260238.81f0db1bea1e11879dc1cafb06c0756d.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-18T18:27:32,268 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 81f0db1bea1e11879dc1cafb06c0756d 2024-11-18T18:27:32,268 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731954260238.81f0db1bea1e11879dc1cafb06c0756d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:27:32,268 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for 81f0db1bea1e11879dc1cafb06c0756d 2024-11-18T18:27:32,268 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for 81f0db1bea1e11879dc1cafb06c0756d 2024-11-18T18:27:32,270 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure c3e2f3fb6516fcaaef4d0a2c8b63842d, server=961a81da9827,38709,1731954257723 because future has completed 2024-11-18T18:27:32,270 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a244d0ab318eb4dd77bc5235828d9d51/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-18T18:27:32,272 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1114): Opened a244d0ab318eb4dd77bc5235828d9d51; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67322831, jitterRate=0.0031883567571640015}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T18:27:32,272 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a244d0ab318eb4dd77bc5235828d9d51 2024-11-18T18:27:32,272 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1006): Region open journal for a244d0ab318eb4dd77bc5235828d9d51: Running coprocessor pre-open hook at 1731954452024Writing region info on filesystem at 1731954452024Initializing all the Stores at 1731954452026 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954452026Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954452030 (+4 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954452030Obtaining lock to block concurrent updates at 1731954452068 (+38 ms)Preparing flush snapshotting stores in a244d0ab318eb4dd77bc5235828d9d51 at 1731954452068Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731954260238.a244d0ab318eb4dd77bc5235828d9d51., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731954452068Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731954260238.a244d0ab318eb4dd77bc5235828d9d51. at 1731954452068Flushing a244d0ab318eb4dd77bc5235828d9d51/colfamily11: creating writer at 1731954452069 (+1 ms)Flushing a244d0ab318eb4dd77bc5235828d9d51/colfamily11: appending metadata at 1731954452129 (+60 ms)Flushing a244d0ab318eb4dd77bc5235828d9d51/colfamily11: closing flushed file at 1731954452129Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70d30aae: reopening flushed file at 1731954452215 (+86 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for a244d0ab318eb4dd77bc5235828d9d51 in 173ms, sequenceid=4, compaction requested=false; wal=null at 1731954452241 (+26 ms)Cleaning up temporary data from old regions at 1731954452251 (+10 ms)Running coprocessor post-open hooks at 1731954452272 (+21 ms)Region opened successfully at 1731954452272 2024-11-18T18:27:32,275 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=76, resume processing ppid=67 2024-11-18T18:27:32,275 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, ppid=67, state=SUCCESS, hasLock=false; OpenRegionProcedure c3e2f3fb6516fcaaef4d0a2c8b63842d, server=961a81da9827,38709,1731954257723 in 879 msec 2024-11-18T18:27:32,276 INFO [StoreOpener-81f0db1bea1e11879dc1cafb06c0756d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 81f0db1bea1e11879dc1cafb06c0756d 2024-11-18T18:27:32,278 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c3e2f3fb6516fcaaef4d0a2c8b63842d, ASSIGN in 1.2160 sec 2024-11-18T18:27:32,278 INFO [StoreOpener-81f0db1bea1e11879dc1cafb06c0756d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81f0db1bea1e11879dc1cafb06c0756d columnFamilyName colfamily11 2024-11-18T18:27:32,278 DEBUG [StoreOpener-81f0db1bea1e11879dc1cafb06c0756d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:27:32,281 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731954260238.a244d0ab318eb4dd77bc5235828d9d51., pid=80, masterSystemTime=1731954451643 2024-11-18T18:27:32,282 INFO [StoreOpener-81f0db1bea1e11879dc1cafb06c0756d-1 {}] regionserver.HStore(327): Store=81f0db1bea1e11879dc1cafb06c0756d/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:27:32,283 INFO [StoreOpener-81f0db1bea1e11879dc1cafb06c0756d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 81f0db1bea1e11879dc1cafb06c0756d 2024-11-18T18:27:32,285 INFO [StoreOpener-81f0db1bea1e11879dc1cafb06c0756d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81f0db1bea1e11879dc1cafb06c0756d columnFamilyName colfamily21 2024-11-18T18:27:32,286 DEBUG [StoreOpener-81f0db1bea1e11879dc1cafb06c0756d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:27:32,287 INFO [StoreOpener-81f0db1bea1e11879dc1cafb06c0756d-1 {}] regionserver.HStore(327): Store=81f0db1bea1e11879dc1cafb06c0756d/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:27:32,287 INFO [StoreOpener-81f0db1bea1e11879dc1cafb06c0756d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 81f0db1bea1e11879dc1cafb06c0756d 2024-11-18T18:27:32,288 INFO [StoreOpener-81f0db1bea1e11879dc1cafb06c0756d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81f0db1bea1e11879dc1cafb06c0756d columnFamilyName colfamily31 2024-11-18T18:27:32,288 DEBUG [StoreOpener-81f0db1bea1e11879dc1cafb06c0756d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:27:32,293 INFO [StoreOpener-81f0db1bea1e11879dc1cafb06c0756d-1 {}] regionserver.HStore(327): Store=81f0db1bea1e11879dc1cafb06c0756d/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:27:32,293 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for 81f0db1bea1e11879dc1cafb06c0756d 2024-11-18T18:27:32,298 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/81f0db1bea1e11879dc1cafb06c0756d 2024-11-18T18:27:32,302 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731954260238.a244d0ab318eb4dd77bc5235828d9d51. 2024-11-18T18:27:32,302 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731954260238.a244d0ab318eb4dd77bc5235828d9d51. 2024-11-18T18:27:32,302 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731954260238.40a213dc6b2a0fd809c71612c437ab83. 2024-11-18T18:27:32,302 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => 40a213dc6b2a0fd809c71612c437ab83, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731954260238.40a213dc6b2a0fd809c71612c437ab83.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-18T18:27:32,303 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 40a213dc6b2a0fd809c71612c437ab83 2024-11-18T18:27:32,303 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731954260238.40a213dc6b2a0fd809c71612c437ab83.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:27:32,303 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for 40a213dc6b2a0fd809c71612c437ab83 2024-11-18T18:27:32,303 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for 40a213dc6b2a0fd809c71612c437ab83 2024-11-18T18:27:32,304 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=a244d0ab318eb4dd77bc5235828d9d51, regionState=OPEN, openSeqNum=5, regionLocation=961a81da9827,42895,1731954257867 2024-11-18T18:27:32,308 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/81f0db1bea1e11879dc1cafb06c0756d 2024-11-18T18:27:32,309 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/81f0db1bea1e11879dc1cafb06c0756d/recovered.edits/0000000000000000004 2024-11-18T18:27:32,314 INFO [StoreOpener-40a213dc6b2a0fd809c71612c437ab83-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 40a213dc6b2a0fd809c71612c437ab83 2024-11-18T18:27:32,317 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=80, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a244d0ab318eb4dd77bc5235828d9d51, server=961a81da9827,42895,1731954257867 because future has completed 2024-11-18T18:27:32,322 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/81f0db1bea1e11879dc1cafb06c0756d/recovered.edits/0000000000000000004 2024-11-18T18:27:32,322 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(2902): Flushing 81f0db1bea1e11879dc1cafb06c0756d 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-18T18:27:32,326 INFO [StoreOpener-40a213dc6b2a0fd809c71612c437ab83-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40a213dc6b2a0fd809c71612c437ab83 columnFamilyName colfamily11 2024-11-18T18:27:32,326 DEBUG [StoreOpener-40a213dc6b2a0fd809c71612c437ab83-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:27:32,330 INFO [StoreOpener-40a213dc6b2a0fd809c71612c437ab83-1 {}] regionserver.HStore(327): Store=40a213dc6b2a0fd809c71612c437ab83/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:27:32,331 INFO [StoreOpener-40a213dc6b2a0fd809c71612c437ab83-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 40a213dc6b2a0fd809c71612c437ab83 2024-11-18T18:27:32,334 INFO [StoreOpener-40a213dc6b2a0fd809c71612c437ab83-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40a213dc6b2a0fd809c71612c437ab83 columnFamilyName colfamily21 2024-11-18T18:27:32,334 DEBUG [StoreOpener-40a213dc6b2a0fd809c71612c437ab83-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:27:32,346 INFO [StoreOpener-40a213dc6b2a0fd809c71612c437ab83-1 {}] regionserver.HStore(327): Store=40a213dc6b2a0fd809c71612c437ab83/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:27:32,346 INFO [StoreOpener-40a213dc6b2a0fd809c71612c437ab83-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 40a213dc6b2a0fd809c71612c437ab83 2024-11-18T18:27:32,350 INFO [StoreOpener-40a213dc6b2a0fd809c71612c437ab83-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40a213dc6b2a0fd809c71612c437ab83 columnFamilyName colfamily31 2024-11-18T18:27:32,350 DEBUG [StoreOpener-40a213dc6b2a0fd809c71612c437ab83-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:27:32,354 INFO [StoreOpener-40a213dc6b2a0fd809c71612c437ab83-1 {}] regionserver.HStore(327): Store=40a213dc6b2a0fd809c71612c437ab83/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:27:32,357 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for 40a213dc6b2a0fd809c71612c437ab83 2024-11-18T18:27:32,358 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=70 2024-11-18T18:27:32,358 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=70, state=SUCCESS, hasLock=false; OpenRegionProcedure a244d0ab318eb4dd77bc5235828d9d51, server=961a81da9827,42895,1731954257867 in 830 msec 2024-11-18T18:27:32,358 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/40a213dc6b2a0fd809c71612c437ab83 2024-11-18T18:27:32,361 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a244d0ab318eb4dd77bc5235828d9d51, ASSIGN in 1.2980 sec 2024-11-18T18:27:32,368 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/40a213dc6b2a0fd809c71612c437ab83 2024-11-18T18:27:32,370 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/40a213dc6b2a0fd809c71612c437ab83/recovered.edits/0000000000000000004 2024-11-18T18:27:32,379 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/81f0db1bea1e11879dc1cafb06c0756d/.tmp/colfamily11/9b0a406ba27e422fa49f41e8d767c766 is 52, key is vvv/colfamily11:colfamily11/1731954264699/Put/seqid=0 2024-11-18T18:27:32,391 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/40a213dc6b2a0fd809c71612c437ab83/recovered.edits/0000000000000000004 2024-11-18T18:27:32,391 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(2902): Flushing 40a213dc6b2a0fd809c71612c437ab83 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-18T18:27:32,459 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/40a213dc6b2a0fd809c71612c437ab83/.tmp/colfamily11/3c981cebf5c04e8991aa6c309d24e010 is 52, key is hhh/colfamily11:colfamily11/1731954264421/Put/seqid=0 2024-11-18T18:27:32,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741884_1060 (size=39263) 2024-11-18T18:27:32,477 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/81f0db1bea1e11879dc1cafb06c0756d/.tmp/colfamily11/9b0a406ba27e422fa49f41e8d767c766 2024-11-18T18:27:32,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741884_1060 (size=39263) 2024-11-18T18:27:32,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741884_1060 (size=39263) 2024-11-18T18:27:32,492 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/81f0db1bea1e11879dc1cafb06c0756d/.tmp/colfamily11/9b0a406ba27e422fa49f41e8d767c766 as hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/81f0db1bea1e11879dc1cafb06c0756d/colfamily11/9b0a406ba27e422fa49f41e8d767c766 2024-11-18T18:27:32,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741885_1061 (size=39263) 2024-11-18T18:27:32,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741885_1061 (size=39263) 2024-11-18T18:27:32,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741885_1061 (size=39263) 2024-11-18T18:27:32,514 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/40a213dc6b2a0fd809c71612c437ab83/.tmp/colfamily11/3c981cebf5c04e8991aa6c309d24e010 2024-11-18T18:27:32,518 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/81f0db1bea1e11879dc1cafb06c0756d/colfamily11/9b0a406ba27e422fa49f41e8d767c766, entries=703, sequenceid=4, filesize=38.3 K 2024-11-18T18:27:32,519 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 81f0db1bea1e11879dc1cafb06c0756d in 196ms, sequenceid=4, compaction requested=false; wal=null 2024-11-18T18:27:32,520 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/81f0db1bea1e11879dc1cafb06c0756d/recovered.edits/0000000000000000004 2024-11-18T18:27:32,521 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for 81f0db1bea1e11879dc1cafb06c0756d 2024-11-18T18:27:32,522 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for 81f0db1bea1e11879dc1cafb06c0756d 2024-11-18T18:27:32,522 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T18:27:32,524 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for 81f0db1bea1e11879dc1cafb06c0756d 2024-11-18T18:27:32,528 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/81f0db1bea1e11879dc1cafb06c0756d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-18T18:27:32,529 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened 81f0db1bea1e11879dc1cafb06c0756d; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74699946, jitterRate=0.11311593651771545}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T18:27:32,529 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 81f0db1bea1e11879dc1cafb06c0756d 2024-11-18T18:27:32,529 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for 81f0db1bea1e11879dc1cafb06c0756d: Running coprocessor pre-open hook at 1731954452268Writing region info on filesystem at 1731954452268Initializing all the Stores at 1731954452269 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954452269Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954452274 (+5 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954452274Obtaining lock to block concurrent updates at 1731954452322 (+48 ms)Preparing flush snapshotting stores in 81f0db1bea1e11879dc1cafb06c0756d at 1731954452322Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731954260238.81f0db1bea1e11879dc1cafb06c0756d., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731954452323 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731954260238.81f0db1bea1e11879dc1cafb06c0756d. at 1731954452323Flushing 81f0db1bea1e11879dc1cafb06c0756d/colfamily11: creating writer at 1731954452323Flushing 81f0db1bea1e11879dc1cafb06c0756d/colfamily11: appending metadata at 1731954452379 (+56 ms)Flushing 81f0db1bea1e11879dc1cafb06c0756d/colfamily11: closing flushed file at 1731954452379Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26a6ee44: reopening flushed file at 1731954452487 (+108 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 81f0db1bea1e11879dc1cafb06c0756d in 196ms, sequenceid=4, compaction requested=false; wal=null at 1731954452519 (+32 ms)Cleaning up temporary data from old regions at 1731954452522 (+3 ms)Running coprocessor post-open hooks at 1731954452529 (+7 ms)Region opened successfully at 1731954452529 2024-11-18T18:27:32,533 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/40a213dc6b2a0fd809c71612c437ab83/.tmp/colfamily11/3c981cebf5c04e8991aa6c309d24e010 as hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/40a213dc6b2a0fd809c71612c437ab83/colfamily11/3c981cebf5c04e8991aa6c309d24e010 2024-11-18T18:27:32,533 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731954260238.81f0db1bea1e11879dc1cafb06c0756d., pid=78, masterSystemTime=1731954451536 2024-11-18T18:27:32,539 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=81f0db1bea1e11879dc1cafb06c0756d, regionState=OPEN, openSeqNum=5, regionLocation=961a81da9827,38709,1731954257723 2024-11-18T18:27:32,541 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731954260238.81f0db1bea1e11879dc1cafb06c0756d. 2024-11-18T18:27:32,541 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731954260238.81f0db1bea1e11879dc1cafb06c0756d. 2024-11-18T18:27:32,544 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 81f0db1bea1e11879dc1cafb06c0756d, server=961a81da9827,38709,1731954257723 because future has completed 2024-11-18T18:27:32,548 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/40a213dc6b2a0fd809c71612c437ab83/colfamily11/3c981cebf5c04e8991aa6c309d24e010, entries=703, sequenceid=4, filesize=38.3 K 2024-11-18T18:27:32,548 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 40a213dc6b2a0fd809c71612c437ab83 in 157ms, sequenceid=4, compaction requested=false; wal=null 2024-11-18T18:27:32,549 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/40a213dc6b2a0fd809c71612c437ab83/recovered.edits/0000000000000000004 2024-11-18T18:27:32,551 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=78, resume processing ppid=71 2024-11-18T18:27:32,551 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, ppid=71, state=SUCCESS, hasLock=false; OpenRegionProcedure 81f0db1bea1e11879dc1cafb06c0756d, server=961a81da9827,38709,1731954257723 in 1.0620 sec 2024-11-18T18:27:32,551 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for 40a213dc6b2a0fd809c71612c437ab83 2024-11-18T18:27:32,551 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for 40a213dc6b2a0fd809c71612c437ab83 2024-11-18T18:27:32,553 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T18:27:32,554 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=81f0db1bea1e11879dc1cafb06c0756d, ASSIGN in 1.4910 sec 2024-11-18T18:27:32,556 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for 40a213dc6b2a0fd809c71612c437ab83 2024-11-18T18:27:32,569 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/40a213dc6b2a0fd809c71612c437ab83/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-18T18:27:32,572 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened 40a213dc6b2a0fd809c71612c437ab83; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65750887, jitterRate=-0.02023543417453766}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T18:27:32,572 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 40a213dc6b2a0fd809c71612c437ab83 2024-11-18T18:27:32,572 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for 40a213dc6b2a0fd809c71612c437ab83: Running coprocessor pre-open hook at 1731954452303Writing region info on filesystem at 1731954452303Initializing all the Stores at 1731954452313 (+10 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954452313Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954452314 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954452314Obtaining lock to block concurrent updates at 1731954452391 (+77 ms)Preparing flush snapshotting stores in 40a213dc6b2a0fd809c71612c437ab83 at 1731954452391Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731954260238.40a213dc6b2a0fd809c71612c437ab83., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731954452392 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731954260238.40a213dc6b2a0fd809c71612c437ab83. at 1731954452392Flushing 40a213dc6b2a0fd809c71612c437ab83/colfamily11: creating writer at 1731954452392Flushing 40a213dc6b2a0fd809c71612c437ab83/colfamily11: appending metadata at 1731954452458 (+66 ms)Flushing 40a213dc6b2a0fd809c71612c437ab83/colfamily11: closing flushed file at 1731954452458Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@fab604a: reopening flushed file at 1731954452523 (+65 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 40a213dc6b2a0fd809c71612c437ab83 in 157ms, sequenceid=4, compaction requested=false; wal=null at 1731954452548 (+25 ms)Cleaning up temporary data from old regions at 1731954452551 (+3 ms)Running coprocessor post-open hooks at 1731954452572 (+21 ms)Region opened successfully at 1731954452572 2024-11-18T18:27:32,574 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731954260238.40a213dc6b2a0fd809c71612c437ab83., pid=79, masterSystemTime=1731954451643 2024-11-18T18:27:32,577 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731954260238.40a213dc6b2a0fd809c71612c437ab83. 2024-11-18T18:27:32,578 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731954260238.40a213dc6b2a0fd809c71612c437ab83. 2024-11-18T18:27:32,578 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731954260238.d1ad74f79ba4d7fd1619c83a55106331. 2024-11-18T18:27:32,578 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7752): Opening region: {ENCODED => d1ad74f79ba4d7fd1619c83a55106331, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731954260238.d1ad74f79ba4d7fd1619c83a55106331.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-18T18:27:32,578 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=40a213dc6b2a0fd809c71612c437ab83, regionState=OPEN, openSeqNum=5, regionLocation=961a81da9827,42895,1731954257867 2024-11-18T18:27:32,579 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true d1ad74f79ba4d7fd1619c83a55106331 2024-11-18T18:27:32,579 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731954260238.d1ad74f79ba4d7fd1619c83a55106331.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:27:32,579 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7794): checking encryption for d1ad74f79ba4d7fd1619c83a55106331 2024-11-18T18:27:32,579 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7797): checking classloading for d1ad74f79ba4d7fd1619c83a55106331 2024-11-18T18:27:32,590 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 40a213dc6b2a0fd809c71612c437ab83, server=961a81da9827,42895,1731954257867 because future has completed 2024-11-18T18:27:32,603 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=68 2024-11-18T18:27:32,603 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=68, state=SUCCESS, hasLock=false; OpenRegionProcedure 40a213dc6b2a0fd809c71612c437ab83, server=961a81da9827,42895,1731954257867 in 1.1090 sec 2024-11-18T18:27:32,606 INFO [StoreOpener-d1ad74f79ba4d7fd1619c83a55106331-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region d1ad74f79ba4d7fd1619c83a55106331 2024-11-18T18:27:32,608 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=40a213dc6b2a0fd809c71612c437ab83, ASSIGN in 1.5430 sec 2024-11-18T18:27:32,612 INFO [StoreOpener-d1ad74f79ba4d7fd1619c83a55106331-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1ad74f79ba4d7fd1619c83a55106331 columnFamilyName colfamily11 2024-11-18T18:27:32,612 DEBUG [StoreOpener-d1ad74f79ba4d7fd1619c83a55106331-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:27:32,616 INFO [StoreOpener-d1ad74f79ba4d7fd1619c83a55106331-1 {}] regionserver.HStore(327): Store=d1ad74f79ba4d7fd1619c83a55106331/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:27:32,616 INFO [StoreOpener-d1ad74f79ba4d7fd1619c83a55106331-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region d1ad74f79ba4d7fd1619c83a55106331 2024-11-18T18:27:32,617 INFO [StoreOpener-d1ad74f79ba4d7fd1619c83a55106331-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1ad74f79ba4d7fd1619c83a55106331 columnFamilyName colfamily21 2024-11-18T18:27:32,618 DEBUG [StoreOpener-d1ad74f79ba4d7fd1619c83a55106331-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:27:32,619 INFO [StoreOpener-d1ad74f79ba4d7fd1619c83a55106331-1 {}] regionserver.HStore(327): Store=d1ad74f79ba4d7fd1619c83a55106331/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:27:32,619 INFO [StoreOpener-d1ad74f79ba4d7fd1619c83a55106331-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region d1ad74f79ba4d7fd1619c83a55106331 2024-11-18T18:27:32,626 INFO [StoreOpener-d1ad74f79ba4d7fd1619c83a55106331-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1ad74f79ba4d7fd1619c83a55106331 columnFamilyName colfamily31 2024-11-18T18:27:32,626 DEBUG [StoreOpener-d1ad74f79ba4d7fd1619c83a55106331-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:27:32,627 INFO [StoreOpener-d1ad74f79ba4d7fd1619c83a55106331-1 {}] regionserver.HStore(327): Store=d1ad74f79ba4d7fd1619c83a55106331/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:27:32,627 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1038): replaying wal for d1ad74f79ba4d7fd1619c83a55106331 2024-11-18T18:27:32,629 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d1ad74f79ba4d7fd1619c83a55106331 2024-11-18T18:27:32,638 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d1ad74f79ba4d7fd1619c83a55106331 2024-11-18T18:27:32,642 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d1ad74f79ba4d7fd1619c83a55106331/recovered.edits/0000000000000000004 2024-11-18T18:27:32,656 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d1ad74f79ba4d7fd1619c83a55106331/recovered.edits/0000000000000000004 2024-11-18T18:27:32,656 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(2902): Flushing d1ad74f79ba4d7fd1619c83a55106331 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-18T18:27:32,703 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d1ad74f79ba4d7fd1619c83a55106331/.tmp/colfamily11/b85739dd83db43aeaa4415d1a6d8a374 is 52, key is uuu/colfamily11:colfamily11/1731954264631/Put/seqid=0 2024-11-18T18:27:32,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741886_1062 (size=39263) 2024-11-18T18:27:32,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741886_1062 (size=39263) 2024-11-18T18:27:32,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741886_1062 (size=39263) 2024-11-18T18:27:32,781 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d1ad74f79ba4d7fd1619c83a55106331/.tmp/colfamily11/b85739dd83db43aeaa4415d1a6d8a374 2024-11-18T18:27:32,795 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d1ad74f79ba4d7fd1619c83a55106331/.tmp/colfamily11/b85739dd83db43aeaa4415d1a6d8a374 as hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d1ad74f79ba4d7fd1619c83a55106331/colfamily11/b85739dd83db43aeaa4415d1a6d8a374 2024-11-18T18:27:32,806 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d1ad74f79ba4d7fd1619c83a55106331/colfamily11/b85739dd83db43aeaa4415d1a6d8a374, entries=703, sequenceid=4, filesize=38.3 K 2024-11-18T18:27:32,806 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for d1ad74f79ba4d7fd1619c83a55106331 in 150ms, sequenceid=4, compaction requested=false; wal=null 2024-11-18T18:27:32,807 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d1ad74f79ba4d7fd1619c83a55106331/recovered.edits/0000000000000000004 2024-11-18T18:27:32,809 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1048): stopping wal replay for d1ad74f79ba4d7fd1619c83a55106331 2024-11-18T18:27:32,809 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1060): Cleaning up temporary data for d1ad74f79ba4d7fd1619c83a55106331 2024-11-18T18:27:32,812 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T18:27:32,815 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1093): writing seq id for d1ad74f79ba4d7fd1619c83a55106331 2024-11-18T18:27:32,825 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d1ad74f79ba4d7fd1619c83a55106331/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-18T18:27:32,827 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1114): Opened d1ad74f79ba4d7fd1619c83a55106331; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74273136, jitterRate=0.10675597190856934}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T18:27:32,827 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d1ad74f79ba4d7fd1619c83a55106331 2024-11-18T18:27:32,827 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1006): Region open journal for d1ad74f79ba4d7fd1619c83a55106331: Running coprocessor pre-open hook at 1731954452579Writing region info on filesystem at 1731954452579Initializing all the Stores at 1731954452590 (+11 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954452590Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954452605 (+15 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954452605Obtaining lock to block concurrent updates at 1731954452656 (+51 ms)Preparing flush snapshotting stores in d1ad74f79ba4d7fd1619c83a55106331 at 1731954452656Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731954260238.d1ad74f79ba4d7fd1619c83a55106331., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731954452656Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731954260238.d1ad74f79ba4d7fd1619c83a55106331. at 1731954452656Flushing d1ad74f79ba4d7fd1619c83a55106331/colfamily11: creating writer at 1731954452656Flushing d1ad74f79ba4d7fd1619c83a55106331/colfamily11: appending metadata at 1731954452699 (+43 ms)Flushing d1ad74f79ba4d7fd1619c83a55106331/colfamily11: closing flushed file at 1731954452699Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31d3b131: reopening flushed file at 1731954452794 (+95 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for d1ad74f79ba4d7fd1619c83a55106331 in 150ms, sequenceid=4, compaction requested=false; wal=null at 1731954452806 (+12 ms)Cleaning up temporary data from old regions at 1731954452809 (+3 ms)Running coprocessor post-open hooks at 1731954452827 (+18 ms)Region opened successfully at 1731954452827 2024-11-18T18:27:32,837 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731954260238.d1ad74f79ba4d7fd1619c83a55106331., pid=81, masterSystemTime=1731954451643 2024-11-18T18:27:32,847 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731954260238.d1ad74f79ba4d7fd1619c83a55106331. 2024-11-18T18:27:32,847 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731954260238.d1ad74f79ba4d7fd1619c83a55106331. 2024-11-18T18:27:32,862 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=d1ad74f79ba4d7fd1619c83a55106331, regionState=OPEN, openSeqNum=5, regionLocation=961a81da9827,42895,1731954257867 2024-11-18T18:27:32,867 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=81, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1ad74f79ba4d7fd1619c83a55106331, server=961a81da9827,42895,1731954257867 because future has completed 2024-11-18T18:27:32,890 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=69 2024-11-18T18:27:32,890 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=69, state=SUCCESS, hasLock=false; OpenRegionProcedure d1ad74f79ba4d7fd1619c83a55106331, server=961a81da9827,42895,1731954257867 in 1.3750 sec 2024-11-18T18:27:32,900 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=57 2024-11-18T18:27:32,900 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d1ad74f79ba4d7fd1619c83a55106331, ASSIGN in 1.8300 sec 2024-11-18T18:27:32,901 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server 961a81da9827,36379,1731954257826 after splitting done 2024-11-18T18:27:32,901 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-18T18:27:32,901 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=SUCCESS, hasLock=true; ServerCrashProcedure 961a81da9827,36379,1731954257826, splitWal=true, meta=true 2024-11-18T18:27:32,901 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T18:27:32,901 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=SUCCESS, hasLock=true; ServerCrashProcedure 961a81da9827,36379,1731954257826, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T18:27:33,061 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 8 exec state=pid=57, state=SUCCESS, hasLock=true; ServerCrashProcedure 961a81da9827,36379,1731954257826, splitWal=true, meta=true 2024-11-18T18:27:33,061 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-18T18:27:33,061 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T18:27:33,061 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-18T18:27:33,061 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-18T18:27:33,062 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-18T18:27:33,062 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-18T18:27:33,062 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-18T18:27:33,062 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-18T18:27:33,066 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 3 msec 2024-11-18T18:27:33,069 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-18T18:27:33,069 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T18:27:33,069 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T18:27:33,069 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T18:27:33,069 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T18:27:33,069 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T18:27:33,069 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T18:27:33,069 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T18:27:33,069 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T18:27:33,069 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T18:27:33,069 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T18:27:33,069 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T18:27:33,069 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T18:27:33,069 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T18:27:33,069 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T18:27:33,069 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T18:27:33,069 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T18:27:33,069 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T18:27:33,069 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T18:27:33,069 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-18T18:27:33,069 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-18T18:27:33,069 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T18:27:33,069 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T18:27:33,069 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-18T18:27:33,069 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-18T18:27:33,070 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-18T18:27:33,070 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-18T18:27:33,077 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_FINISH, hasLock=false; ServerCrashProcedure 961a81da9827,36379,1731954257826, splitWal=true, meta=true 2024-11-18T18:27:33,077 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 961a81da9827%2C36379%2C1731954257826.meta.1731954259788.meta, worker=961a81da9827,42895,1731954257867 2024-11-18T18:27:33,077 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 961a81da9827%2C36379%2C1731954257826.meta.1731954259788.meta, worker=961a81da9827,42895,1731954257867 2024-11-18T18:27:33,077 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T18:27:33,078 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=961a81da9827,42895,1731954257867 2024-11-18T18:27:33,078 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 961a81da9827%2C36379%2C1731954257826.1731954259295, worker=961a81da9827,42895,1731954257867 2024-11-18T18:27:33,078 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 961a81da9827%2C36379%2C1731954257826.1731954259295, worker=961a81da9827,42895,1731954257867 2024-11-18T18:27:33,078 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5e69ef2147d87d1b83dbcbb4705cbabe, ASSIGN 2024-11-18T18:27:33,078 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=bcf8695060a326b7d24615ba6cb932b1, ASSIGN 2024-11-18T18:27:33,078 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=43138a6a12c2efea7093064622bb3cd8, ASSIGN 2024-11-18T18:27:33,078 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c3e2f3fb6516fcaaef4d0a2c8b63842d, ASSIGN 2024-11-18T18:27:33,078 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=40a213dc6b2a0fd809c71612c437ab83, ASSIGN 2024-11-18T18:27:33,078 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d1ad74f79ba4d7fd1619c83a55106331, ASSIGN 2024-11-18T18:27:33,078 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a244d0ab318eb4dd77bc5235828d9d51, ASSIGN 2024-11-18T18:27:33,078 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=81f0db1bea1e11879dc1cafb06c0756d, ASSIGN 2024-11-18T18:27:33,078 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=779798cd60db955ed78a4329cac08969, ASSIGN 2024-11-18T18:27:33,078 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=72, state=SUCCESS, hasLock=false; OpenRegionProcedure 779798cd60db955ed78a4329cac08969, server=961a81da9827,42895,1731954257867 2024-11-18T18:27:33,078 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=66, state=SUCCESS, hasLock=false; OpenRegionProcedure 43138a6a12c2efea7093064622bb3cd8, server=961a81da9827,42895,1731954257867 2024-11-18T18:27:33,078 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=65, state=SUCCESS, hasLock=false; OpenRegionProcedure bcf8695060a326b7d24615ba6cb932b1, server=961a81da9827,38709,1731954257723 2024-11-18T18:27:33,078 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=67, state=SUCCESS, hasLock=false; OpenRegionProcedure c3e2f3fb6516fcaaef4d0a2c8b63842d, server=961a81da9827,38709,1731954257723 2024-11-18T18:27:33,078 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure 5e69ef2147d87d1b83dbcbb4705cbabe, server=961a81da9827,38709,1731954257723 2024-11-18T18:27:33,078 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=71, state=SUCCESS, hasLock=false; OpenRegionProcedure 81f0db1bea1e11879dc1cafb06c0756d, server=961a81da9827,38709,1731954257723 2024-11-18T18:27:33,078 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=68, state=SUCCESS, hasLock=false; OpenRegionProcedure 40a213dc6b2a0fd809c71612c437ab83, server=961a81da9827,42895,1731954257867 2024-11-18T18:27:33,078 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=70, state=SUCCESS, hasLock=false; OpenRegionProcedure a244d0ab318eb4dd77bc5235828d9d51, server=961a81da9827,42895,1731954257867 2024-11-18T18:27:33,078 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=69, state=SUCCESS, hasLock=false; OpenRegionProcedure d1ad74f79ba4d7fd1619c83a55106331, server=961a81da9827,42895,1731954257867 2024-11-18T18:27:33,078 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_FINISH, hasLock=false; ServerCrashProcedure 961a81da9827,36379,1731954257826, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-18T18:27:33,078 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 961a81da9827%2C36379%2C1731954257826.meta.1731954259788.meta, worker=961a81da9827,42895,1731954257867 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T18:27:33,078 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 961a81da9827%2C36379%2C1731954257826.meta.1731954259788.meta, worker=961a81da9827,42895,1731954257867 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T18:27:33,078 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T18:27:33,078 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=961a81da9827,42895,1731954257867 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T18:27:33,079 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 961a81da9827%2C36379%2C1731954257826.1731954259295, worker=961a81da9827,42895,1731954257867 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T18:27:33,079 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 961a81da9827%2C36379%2C1731954257826.1731954259295, worker=961a81da9827,42895,1731954257867 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T18:27:33,079 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5e69ef2147d87d1b83dbcbb4705cbabe, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T18:27:33,079 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=bcf8695060a326b7d24615ba6cb932b1, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T18:27:33,079 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=43138a6a12c2efea7093064622bb3cd8, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T18:27:33,079 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=67, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c3e2f3fb6516fcaaef4d0a2c8b63842d, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T18:27:33,079 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=68, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=40a213dc6b2a0fd809c71612c437ab83, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T18:27:33,079 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=69, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d1ad74f79ba4d7fd1619c83a55106331, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T18:27:33,079 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=70, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a244d0ab318eb4dd77bc5235828d9d51, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T18:27:33,079 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=71, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=81f0db1bea1e11879dc1cafb06c0756d, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T18:27:33,079 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=779798cd60db955ed78a4329cac08969, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T18:27:33,079 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=72, state=SUCCESS, hasLock=false; OpenRegionProcedure 779798cd60db955ed78a4329cac08969, server=961a81da9827,42895,1731954257867 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T18:27:33,079 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=66, state=SUCCESS, hasLock=false; OpenRegionProcedure 43138a6a12c2efea7093064622bb3cd8, server=961a81da9827,42895,1731954257867 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T18:27:33,079 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=65, state=SUCCESS, hasLock=false; OpenRegionProcedure bcf8695060a326b7d24615ba6cb932b1, server=961a81da9827,38709,1731954257723 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T18:27:33,079 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=67, state=SUCCESS, hasLock=false; OpenRegionProcedure c3e2f3fb6516fcaaef4d0a2c8b63842d, server=961a81da9827,38709,1731954257723 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T18:27:33,079 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure 5e69ef2147d87d1b83dbcbb4705cbabe, server=961a81da9827,38709,1731954257723 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T18:27:33,079 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=71, state=SUCCESS, hasLock=false; OpenRegionProcedure 81f0db1bea1e11879dc1cafb06c0756d, server=961a81da9827,38709,1731954257723 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T18:27:33,079 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=68, state=SUCCESS, hasLock=false; OpenRegionProcedure 40a213dc6b2a0fd809c71612c437ab83, server=961a81da9827,42895,1731954257867 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T18:27:33,079 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=70, state=SUCCESS, hasLock=false; OpenRegionProcedure a244d0ab318eb4dd77bc5235828d9d51, server=961a81da9827,42895,1731954257867 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T18:27:33,079 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=69, state=SUCCESS, hasLock=false; OpenRegionProcedure d1ad74f79ba4d7fd1619c83a55106331, server=961a81da9827,42895,1731954257867 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T18:27:33,079 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 13 msec 2024-11-18T18:27:33,080 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=61 2024-11-18T18:27:33,080 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=961a81da9827,42895,1731954257867, table=hbase:meta, region=1588230740 2024-11-18T18:27:33,081 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 961a81da9827,42895,1731954257867, state=OPEN 2024-11-18T18:27:33,087 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33089-0x10107179d9f0000, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T18:27:33,087 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:27:33,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38709-0x10107179d9f0001, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T18:27:33,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42895-0x10107179d9f0003, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T18:27:33,090 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:27:33,090 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:27:33,106 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731954483105 2024-11-18T18:27:33,106 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-18T18:27:33,115 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8bb7ae0747eda2290799ca8fa83f9ede, regionState=OPEN, lastHost=961a81da9827,38709,1731954257723, regionLocation=961a81da9827,38709,1731954257723, openSeqNum=2 2024-11-18T18:27:33,115 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22971deb0db38b7cccde5c3bc0868f22, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=2 2024-11-18T18:27:33,115 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43138a6a12c2efea7093064622bb3cd8, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=5 2024-11-18T18:27:33,116 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb099f0975ebdb618ef8e3b2c263bbca, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=2 2024-11-18T18:27:33,116 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bcf8695060a326b7d24615ba6cb932b1, regionState=OPEN, lastHost=961a81da9827,38709,1731954257723, regionLocation=961a81da9827,38709,1731954257723, openSeqNum=5 2024-11-18T18:27:33,116 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb91ff28ef3c18f0dc0d3f26aac68eca, regionState=OPEN, lastHost=961a81da9827,38709,1731954257723, regionLocation=961a81da9827,38709,1731954257723, openSeqNum=2 2024-11-18T18:27:33,116 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8b4dd2a6d8dd59085c547b55aac0de6, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=2 2024-11-18T18:27:33,116 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40a213dc6b2a0fd809c71612c437ab83, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=5 2024-11-18T18:27:33,116 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f39571a93e25091090ea0831d54936b5, regionState=OPEN, lastHost=961a81da9827,38709,1731954257723, regionLocation=961a81da9827,38709,1731954257723, openSeqNum=2 2024-11-18T18:27:33,116 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97a938e093c14701871e85a1fbb6e07d, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=2 2024-11-18T18:27:33,116 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c3e2f3fb6516fcaaef4d0a2c8b63842d, regionState=OPEN, lastHost=961a81da9827,38709,1731954257723, regionLocation=961a81da9827,38709,1731954257723, openSeqNum=5 2024-11-18T18:27:33,116 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97377a91d35f8b242120f603c456c41e, regionState=OPEN, lastHost=961a81da9827,38709,1731954257723, regionLocation=961a81da9827,38709,1731954257723, openSeqNum=2 2024-11-18T18:27:33,116 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a244d0ab318eb4dd77bc5235828d9d51, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=5 2024-11-18T18:27:33,116 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=012c277c99c564dd90e8e5f6850b6ce0, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=2 2024-11-18T18:27:33,117 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=140f5077d3b730b643b558ebe8a13abd, regionState=OPEN, lastHost=961a81da9827,38709,1731954257723, regionLocation=961a81da9827,38709,1731954257723, openSeqNum=2 2024-11-18T18:27:33,117 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e69ef2147d87d1b83dbcbb4705cbabe, regionState=OPEN, lastHost=961a81da9827,38709,1731954257723, regionLocation=961a81da9827,38709,1731954257723, openSeqNum=5 2024-11-18T18:27:33,117 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb8fa4293ea5e58c18dbc5b978355ca9, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=2 2024-11-18T18:27:33,117 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69ae79c6f9eeed559fa88701f773706c, regionState=OPEN, lastHost=961a81da9827,38709,1731954257723, regionLocation=961a81da9827,38709,1731954257723, openSeqNum=2 2024-11-18T18:27:33,117 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=75a13ed51395b1016b90c6b0d90a7d0a, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=2 2024-11-18T18:27:33,117 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aaf2435418703b2a31c1190af1560ba8, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=2 2024-11-18T18:27:33,117 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1ad74f79ba4d7fd1619c83a55106331, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=5 2024-11-18T18:27:33,117 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=81f0db1bea1e11879dc1cafb06c0756d, regionState=OPEN, lastHost=961a81da9827,38709,1731954257723, regionLocation=961a81da9827,38709,1731954257723, openSeqNum=5 2024-11-18T18:27:33,117 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2a8fcaf8905dcde0537806fb16b9ff4c, regionState=OPEN, lastHost=961a81da9827,38709,1731954257723, regionLocation=961a81da9827,38709,1731954257723, openSeqNum=2 2024-11-18T18:27:33,118 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=579ee42a94597fdc1db9a4820a53ee64, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=2 2024-11-18T18:27:33,118 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=779798cd60db955ed78a4329cac08969, regionState=OPEN, lastHost=961a81da9827,42895,1731954257867, regionLocation=961a81da9827,42895,1731954257867, openSeqNum=5 2024-11-18T18:27:33,118 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=08ccc32db22fb75f5466ad774ad9e8ba, regionState=OPEN, lastHost=961a81da9827,38709,1731954257723, regionLocation=961a81da9827,38709,1731954257723, openSeqNum=2 2024-11-18T18:27:33,118 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-18T18:27:33,118 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731954513118 2024-11-18T18:27:33,118 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731954573118 2024-11-18T18:27:33,118 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 12 msec 2024-11-18T18:27:33,118 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server 961a81da9827,36379,1731954257826 after splitting done 2024-11-18T18:27:33,118 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-18T18:27:33,123 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; ServerCrashProcedure 961a81da9827,36379,1731954257826, splitWal=true, meta=true in 3 mins, 7.649 sec 2024-11-18T18:27:33,375 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42100, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T18:27:33,379 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33704, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T18:27:33,387 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57248, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T18:27:33,393 WARN [RPCClient-NioEventLoopGroup-6-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 961a81da9827:36379 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 961a81da9827/172.17.0.2:36379 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T18:27:33,395 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731954260238.43138a6a12c2efea7093064622bb3cd8., hostname=961a81da9827,36379,1731954257826, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731954260238.43138a6a12c2efea7093064622bb3cd8., hostname=961a81da9827,36379,1731954257826, seqNum=2, error=java.net.ConnectException: Call to address=961a81da9827:36379 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:27:33,395 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731954260238.43138a6a12c2efea7093064622bb3cd8., hostname=961a81da9827,36379,1731954257826, seqNum=2 is java.net.ConnectException: Connection refused 2024-11-18T18:27:33,395 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731954260238.43138a6a12c2efea7093064622bb3cd8., hostname=961a81da9827,36379,1731954257826, seqNum=2 from cache 2024-11-18T18:27:33,395 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.FailedServers(52): Added failed server with address 961a81da9827:36379 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 961a81da9827/172.17.0.2:36379 2024-11-18T18:27:33,503 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 961a81da9827:36379 this server is in the failed servers list 2024-11-18T18:27:33,504 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=961a81da9827:36379 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:27:33,504 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:27:33,504 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=961a81da9827,36379,1731954257826, seqNum=-1 from cache 2024-11-18T18:27:33,613 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T18:27:33,614 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42110, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T18:27:33,615 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=961a81da9827,42895,1731954257867, seqNum=-1] 2024-11-18T18:27:33,620 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ccc', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731954260238.43138a6a12c2efea7093064622bb3cd8., hostname=961a81da9827,42895,1731954257867, seqNum=5] 2024-11-18T18:27:33,644 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to 961a81da9827:36379 this server is in the failed servers list 2024-11-18T18:27:33,645 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731954260238.bcf8695060a326b7d24615ba6cb932b1., hostname=961a81da9827,36379,1731954257826, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731954260238.bcf8695060a326b7d24615ba6cb932b1., hostname=961a81da9827,36379,1731954257826, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=961a81da9827:36379 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:27:33,645 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731954260238.bcf8695060a326b7d24615ba6cb932b1., hostname=961a81da9827,36379,1731954257826, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:27:33,645 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731954260238.bcf8695060a326b7d24615ba6cb932b1., hostname=961a81da9827,36379,1731954257826, seqNum=2 from cache 2024-11-18T18:27:33,763 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eee', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731954260238.bcf8695060a326b7d24615ba6cb932b1., hostname=961a81da9827,38709,1731954257723, seqNum=5] 2024-11-18T18:27:33,802 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to 961a81da9827:36379 this server is in the failed servers list 2024-11-18T18:27:33,803 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731954260238.40a213dc6b2a0fd809c71612c437ab83., hostname=961a81da9827,36379,1731954257826, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731954260238.40a213dc6b2a0fd809c71612c437ab83., hostname=961a81da9827,36379,1731954257826, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=961a81da9827:36379 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:27:33,803 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731954260238.40a213dc6b2a0fd809c71612c437ab83., hostname=961a81da9827,36379,1731954257826, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:27:33,804 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731954260238.40a213dc6b2a0fd809c71612c437ab83., hostname=961a81da9827,36379,1731954257826, seqNum=2 from cache 2024-11-18T18:27:33,922 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhh', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731954260238.40a213dc6b2a0fd809c71612c437ab83., hostname=961a81da9827,42895,1731954257867, seqNum=5] 2024-11-18T18:27:33,950 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to 961a81da9827:36379 this server is in the failed servers list 2024-11-18T18:27:33,950 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731954260238.c3e2f3fb6516fcaaef4d0a2c8b63842d., hostname=961a81da9827,36379,1731954257826, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731954260238.c3e2f3fb6516fcaaef4d0a2c8b63842d., hostname=961a81da9827,36379,1731954257826, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=961a81da9827:36379 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:27:33,951 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731954260238.c3e2f3fb6516fcaaef4d0a2c8b63842d., hostname=961a81da9827,36379,1731954257826, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:27:33,951 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731954260238.c3e2f3fb6516fcaaef4d0a2c8b63842d., hostname=961a81da9827,36379,1731954257826, seqNum=2 from cache 2024-11-18T18:27:34,060 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkk', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731954260238.c3e2f3fb6516fcaaef4d0a2c8b63842d., hostname=961a81da9827,38709,1731954257723, seqNum=5] 2024-11-18T18:27:34,080 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.AbstractRpcClient(357): Not trying to connect to 961a81da9827:36379 this server is in the failed servers list 2024-11-18T18:27:34,081 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731954260238.a244d0ab318eb4dd77bc5235828d9d51., hostname=961a81da9827,36379,1731954257826, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731954260238.a244d0ab318eb4dd77bc5235828d9d51., hostname=961a81da9827,36379,1731954257826, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=961a81da9827:36379 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:27:34,081 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731954260238.a244d0ab318eb4dd77bc5235828d9d51., hostname=961a81da9827,36379,1731954257826, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:27:34,081 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731954260238.a244d0ab318eb4dd77bc5235828d9d51., hostname=961a81da9827,36379,1731954257826, seqNum=2 from cache 2024-11-18T18:27:34,188 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731954260238.a244d0ab318eb4dd77bc5235828d9d51., hostname=961a81da9827,42895,1731954257867, seqNum=5] 2024-11-18T18:27:34,208 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.AbstractRpcClient(357): Not trying to connect to 961a81da9827:36379 this server is in the failed servers list 2024-11-18T18:27:34,208 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731954260238.5e69ef2147d87d1b83dbcbb4705cbabe., hostname=961a81da9827,36379,1731954257826, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731954260238.5e69ef2147d87d1b83dbcbb4705cbabe., hostname=961a81da9827,36379,1731954257826, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=961a81da9827:36379 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:27:34,208 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731954260238.5e69ef2147d87d1b83dbcbb4705cbabe., hostname=961a81da9827,36379,1731954257826, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:27:34,208 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731954260238.5e69ef2147d87d1b83dbcbb4705cbabe., hostname=961a81da9827,36379,1731954257826, seqNum=2 from cache 2024-11-18T18:27:34,318 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731954260238.5e69ef2147d87d1b83dbcbb4705cbabe., hostname=961a81da9827,38709,1731954257723, seqNum=5] 2024-11-18T18:27:34,337 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to 961a81da9827:36379 this server is in the failed servers list 2024-11-18T18:27:34,337 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731954260238.d1ad74f79ba4d7fd1619c83a55106331., hostname=961a81da9827,36379,1731954257826, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731954260238.d1ad74f79ba4d7fd1619c83a55106331., hostname=961a81da9827,36379,1731954257826, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=961a81da9827:36379 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:27:34,337 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731954260238.d1ad74f79ba4d7fd1619c83a55106331., hostname=961a81da9827,36379,1731954257826, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:27:34,337 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731954260238.d1ad74f79ba4d7fd1619c83a55106331., hostname=961a81da9827,36379,1731954257826, seqNum=2 from cache 2024-11-18T18:27:34,447 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uuu', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731954260238.d1ad74f79ba4d7fd1619c83a55106331., hostname=961a81da9827,42895,1731954257867, seqNum=5] 2024-11-18T18:27:34,455 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to 961a81da9827:36379 this server is in the failed servers list 2024-11-18T18:27:34,455 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731954260238.81f0db1bea1e11879dc1cafb06c0756d., hostname=961a81da9827,36379,1731954257826, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731954260238.81f0db1bea1e11879dc1cafb06c0756d., hostname=961a81da9827,36379,1731954257826, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=961a81da9827:36379 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:27:34,455 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731954260238.81f0db1bea1e11879dc1cafb06c0756d., hostname=961a81da9827,36379,1731954257826, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:27:34,455 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731954260238.81f0db1bea1e11879dc1cafb06c0756d., hostname=961a81da9827,36379,1731954257826, seqNum=2 from cache 2024-11-18T18:27:34,568 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731954260238.81f0db1bea1e11879dc1cafb06c0756d., hostname=961a81da9827,38709,1731954257723, seqNum=5] 2024-11-18T18:27:34,587 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to 961a81da9827:36379 this server is in the failed servers list 2024-11-18T18:27:34,587 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731954260238.779798cd60db955ed78a4329cac08969., hostname=961a81da9827,36379,1731954257826, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731954260238.779798cd60db955ed78a4329cac08969., hostname=961a81da9827,36379,1731954257826, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=961a81da9827:36379 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:27:34,587 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731954260238.779798cd60db955ed78a4329cac08969., hostname=961a81da9827,36379,1731954257826, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 961a81da9827:36379 2024-11-18T18:27:34,588 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731954260238.779798cd60db955ed78a4329cac08969., hostname=961a81da9827,36379,1731954257826, seqNum=2 from cache 2024-11-18T18:27:34,698 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yyy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731954260238.779798cd60db955ed78a4329cac08969., hostname=961a81da9827,42895,1731954257867, seqNum=5] 2024-11-18T18:27:34,795 WARN [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false 2024-11-18T18:27:34,795 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-18T18:27:34,795 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T18:27:34,796 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.procedure.TestSCPBase.tearDown(TestSCPBase.java:72) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T18:27:34,796 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T18:27:34,796 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T18:27:34,796 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T18:27:34,797 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-18T18:27:34,797 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1930243127, stopped=false 2024-11-18T18:27:34,797 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=961a81da9827,33089,1731954257001 2024-11-18T18:27:34,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42895-0x10107179d9f0003, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-18T18:27:34,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42895-0x10107179d9f0003, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:27:34,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38709-0x10107179d9f0001, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-18T18:27:34,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33089-0x10107179d9f0000, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-18T18:27:34,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38709-0x10107179d9f0001, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:27:34,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33089-0x10107179d9f0000, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:27:34,800 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T18:27:34,800 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T18:27:34,800 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.procedure.TestSCPBase.tearDown(TestSCPBase.java:72) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T18:27:34,800 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T18:27:34,801 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '961a81da9827,38709,1731954257723' ***** 2024-11-18T18:27:34,801 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-18T18:27:34,801 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '961a81da9827,42895,1731954257867' ***** 2024-11-18T18:27:34,801 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-18T18:27:34,801 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38709-0x10107179d9f0001, quorum=127.0.0.1:59292, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T18:27:34,801 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33089-0x10107179d9f0000, quorum=127.0.0.1:59292, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T18:27:34,801 INFO [RS:0;961a81da9827:38709 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-18T18:27:34,801 INFO [RS:2;961a81da9827:42895 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-18T18:27:34,802 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-18T18:27:34,802 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42895-0x10107179d9f0003, quorum=127.0.0.1:59292, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T18:27:34,802 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-18T18:27:34,802 INFO [RS:2;961a81da9827:42895 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-18T18:27:34,802 INFO [RS:0;961a81da9827:38709 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-18T18:27:34,802 INFO [RS:2;961a81da9827:42895 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-18T18:27:34,802 INFO [RS:0;961a81da9827:38709 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-18T18:27:34,803 INFO [RS:2;961a81da9827:42895 {}] regionserver.HRegionServer(3091): Received CLOSE for 75a13ed51395b1016b90c6b0d90a7d0a 2024-11-18T18:27:34,803 INFO [RS:0;961a81da9827:38709 {}] regionserver.HRegionServer(3091): Received CLOSE for 81f0db1bea1e11879dc1cafb06c0756d 2024-11-18T18:27:34,805 INFO [RS:2;961a81da9827:42895 {}] regionserver.HRegionServer(3091): Received CLOSE for 012c277c99c564dd90e8e5f6850b6ce0 2024-11-18T18:27:34,805 INFO [RS:2;961a81da9827:42895 {}] regionserver.HRegionServer(3091): Received CLOSE for 22971deb0db38b7cccde5c3bc0868f22 2024-11-18T18:27:34,805 INFO [RS:2;961a81da9827:42895 {}] regionserver.HRegionServer(3091): Received CLOSE for 579ee42a94597fdc1db9a4820a53ee64 2024-11-18T18:27:34,805 INFO [RS:2;961a81da9827:42895 {}] regionserver.HRegionServer(3091): Received CLOSE for 40a213dc6b2a0fd809c71612c437ab83 2024-11-18T18:27:34,805 INFO [RS:2;961a81da9827:42895 {}] regionserver.HRegionServer(3091): Received CLOSE for 779798cd60db955ed78a4329cac08969 2024-11-18T18:27:34,805 INFO [RS:2;961a81da9827:42895 {}] regionserver.HRegionServer(3091): Received CLOSE for a244d0ab318eb4dd77bc5235828d9d51 2024-11-18T18:27:34,805 INFO [RS:2;961a81da9827:42895 {}] regionserver.HRegionServer(3091): Received CLOSE for bb099f0975ebdb618ef8e3b2c263bbca 2024-11-18T18:27:34,805 INFO [RS:2;961a81da9827:42895 {}] regionserver.HRegionServer(3091): Received CLOSE for d1ad74f79ba4d7fd1619c83a55106331 2024-11-18T18:27:34,805 INFO [RS:2;961a81da9827:42895 {}] regionserver.HRegionServer(3091): Received CLOSE for c8b4dd2a6d8dd59085c547b55aac0de6 2024-11-18T18:27:34,805 INFO [RS:2;961a81da9827:42895 {}] regionserver.HRegionServer(3091): Received CLOSE for fb8fa4293ea5e58c18dbc5b978355ca9 2024-11-18T18:27:34,805 INFO [RS:2;961a81da9827:42895 {}] regionserver.HRegionServer(3091): Received CLOSE for 97a938e093c14701871e85a1fbb6e07d 2024-11-18T18:27:34,805 INFO [RS:2;961a81da9827:42895 {}] regionserver.HRegionServer(3091): Received CLOSE for 43138a6a12c2efea7093064622bb3cd8 2024-11-18T18:27:34,805 INFO [RS:2;961a81da9827:42895 {}] regionserver.HRegionServer(3091): Received CLOSE for aaf2435418703b2a31c1190af1560ba8 2024-11-18T18:27:34,806 INFO [RS:2;961a81da9827:42895 {}] regionserver.HRegionServer(959): stopping server 961a81da9827,42895,1731954257867 2024-11-18T18:27:34,806 INFO [RS:2;961a81da9827:42895 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T18:27:34,806 INFO [RS:2;961a81da9827:42895 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;961a81da9827:42895. 2024-11-18T18:27:34,806 DEBUG [RS:2;961a81da9827:42895 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T18:27:34,806 DEBUG [RS:2;961a81da9827:42895 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T18:27:34,806 INFO [RS:2;961a81da9827:42895 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-18T18:27:34,806 INFO [RS:2;961a81da9827:42895 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-18T18:27:34,806 INFO [RS:2;961a81da9827:42895 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-18T18:27:34,807 INFO [RS:2;961a81da9827:42895 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-18T18:27:34,807 INFO [RS:0;961a81da9827:38709 {}] regionserver.HRegionServer(3091): Received CLOSE for fb91ff28ef3c18f0dc0d3f26aac68eca 2024-11-18T18:27:34,807 INFO [RS:0;961a81da9827:38709 {}] regionserver.HRegionServer(3091): Received CLOSE for 140f5077d3b730b643b558ebe8a13abd 2024-11-18T18:27:34,807 INFO [RS:0;961a81da9827:38709 {}] regionserver.HRegionServer(3091): Received CLOSE for 8bb7ae0747eda2290799ca8fa83f9ede 2024-11-18T18:27:34,807 INFO [RS:0;961a81da9827:38709 {}] regionserver.HRegionServer(3091): Received CLOSE for c3e2f3fb6516fcaaef4d0a2c8b63842d 2024-11-18T18:27:34,807 INFO [RS:0;961a81da9827:38709 {}] regionserver.HRegionServer(3091): Received CLOSE for 5e69ef2147d87d1b83dbcbb4705cbabe 2024-11-18T18:27:34,807 INFO [RS:0;961a81da9827:38709 {}] regionserver.HRegionServer(3091): Received CLOSE for 2a8fcaf8905dcde0537806fb16b9ff4c 2024-11-18T18:27:34,807 INFO [RS:0;961a81da9827:38709 {}] regionserver.HRegionServer(3091): Received CLOSE for 08ccc32db22fb75f5466ad774ad9e8ba 2024-11-18T18:27:34,807 INFO [RS:0;961a81da9827:38709 {}] regionserver.HRegionServer(3091): Received CLOSE for 97377a91d35f8b242120f603c456c41e 2024-11-18T18:27:34,807 INFO [RS:0;961a81da9827:38709 {}] regionserver.HRegionServer(3091): Received CLOSE for 69ae79c6f9eeed559fa88701f773706c 2024-11-18T18:27:34,807 INFO [RS:0;961a81da9827:38709 {}] regionserver.HRegionServer(3091): Received CLOSE for bcf8695060a326b7d24615ba6cb932b1 2024-11-18T18:27:34,807 INFO [RS:0;961a81da9827:38709 {}] regionserver.HRegionServer(3091): Received CLOSE for f39571a93e25091090ea0831d54936b5 2024-11-18T18:27:34,807 INFO [RS:0;961a81da9827:38709 {}] regionserver.HRegionServer(959): stopping server 961a81da9827,38709,1731954257723 2024-11-18T18:27:34,807 INFO [RS:0;961a81da9827:38709 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T18:27:34,808 INFO [RS:0;961a81da9827:38709 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;961a81da9827:38709. 2024-11-18T18:27:34,808 DEBUG [RS:0;961a81da9827:38709 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T18:27:34,808 DEBUG [RS:0;961a81da9827:38709 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T18:27:34,808 INFO [RS:0;961a81da9827:38709 {}] regionserver.HRegionServer(1321): Waiting on 12 regions to close 2024-11-18T18:27:34,808 DEBUG [RS:0;961a81da9827:38709 {}] regionserver.HRegionServer(1351): Waiting on 08ccc32db22fb75f5466ad774ad9e8ba, 140f5077d3b730b643b558ebe8a13abd, 2a8fcaf8905dcde0537806fb16b9ff4c, 5e69ef2147d87d1b83dbcbb4705cbabe, 69ae79c6f9eeed559fa88701f773706c, 81f0db1bea1e11879dc1cafb06c0756d, 8bb7ae0747eda2290799ca8fa83f9ede, 97377a91d35f8b242120f603c456c41e, bcf8695060a326b7d24615ba6cb932b1, c3e2f3fb6516fcaaef4d0a2c8b63842d, f39571a93e25091090ea0831d54936b5, fb91ff28ef3c18f0dc0d3f26aac68eca 2024-11-18T18:27:34,809 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 81f0db1bea1e11879dc1cafb06c0756d, disabling compactions & flushes 2024-11-18T18:27:34,810 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731954260238.81f0db1bea1e11879dc1cafb06c0756d. 2024-11-18T18:27:34,810 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731954260238.81f0db1bea1e11879dc1cafb06c0756d. 2024-11-18T18:27:34,810 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731954260238.81f0db1bea1e11879dc1cafb06c0756d. after waiting 0 ms 2024-11-18T18:27:34,810 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731954260238.81f0db1bea1e11879dc1cafb06c0756d. 2024-11-18T18:27:34,811 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 75a13ed51395b1016b90c6b0d90a7d0a, disabling compactions & flushes 2024-11-18T18:27:34,811 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731954260238.75a13ed51395b1016b90c6b0d90a7d0a. 2024-11-18T18:27:34,811 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731954260238.75a13ed51395b1016b90c6b0d90a7d0a. 2024-11-18T18:27:34,811 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731954260238.75a13ed51395b1016b90c6b0d90a7d0a. after waiting 0 ms 2024-11-18T18:27:34,811 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731954260238.75a13ed51395b1016b90c6b0d90a7d0a. 2024-11-18T18:27:34,811 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 75a13ed51395b1016b90c6b0d90a7d0a 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-18T18:27:34,815 INFO [RS:2;961a81da9827:42895 {}] regionserver.HRegionServer(1321): Waiting on 15 regions to close 2024-11-18T18:27:34,815 DEBUG [RS:2;961a81da9827:42895 {}] regionserver.HRegionServer(1351): Waiting on 012c277c99c564dd90e8e5f6850b6ce0, 1588230740, 22971deb0db38b7cccde5c3bc0868f22, 40a213dc6b2a0fd809c71612c437ab83, 43138a6a12c2efea7093064622bb3cd8, 579ee42a94597fdc1db9a4820a53ee64, 75a13ed51395b1016b90c6b0d90a7d0a, 779798cd60db955ed78a4329cac08969, 97a938e093c14701871e85a1fbb6e07d, a244d0ab318eb4dd77bc5235828d9d51, aaf2435418703b2a31c1190af1560ba8, bb099f0975ebdb618ef8e3b2c263bbca, c8b4dd2a6d8dd59085c547b55aac0de6, d1ad74f79ba4d7fd1619c83a55106331, fb8fa4293ea5e58c18dbc5b978355ca9 2024-11-18T18:27:34,816 DEBUG [RS_CLOSE_META-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-18T18:27:34,816 INFO [RS_CLOSE_META-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-18T18:27:34,816 DEBUG [RS_CLOSE_META-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-18T18:27:34,816 DEBUG [RS_CLOSE_META-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-18T18:27:34,816 DEBUG [RS_CLOSE_META-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-18T18:27:34,817 INFO [RS_CLOSE_META-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=13.39 KB heapSize=19.70 KB 2024-11-18T18:27:34,831 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/81f0db1bea1e11879dc1cafb06c0756d/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-18T18:27:34,838 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731954260238.81f0db1bea1e11879dc1cafb06c0756d. 2024-11-18T18:27:34,838 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 81f0db1bea1e11879dc1cafb06c0756d: Waiting for close lock at 1731954454809Running coprocessor pre-close hooks at 1731954454809Disabling compacts and flushes for region at 1731954454809Disabling writes for close at 1731954454810 (+1 ms)Writing region close event to WAL at 1731954454818 (+8 ms)Running coprocessor post-close hooks at 1731954454832 (+14 ms)Closed at 1731954454838 (+6 ms) 2024-11-18T18:27:34,839 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731954260238.81f0db1bea1e11879dc1cafb06c0756d. 2024-11-18T18:27:34,839 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fb91ff28ef3c18f0dc0d3f26aac68eca, disabling compactions & flushes 2024-11-18T18:27:34,839 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731954260238.fb91ff28ef3c18f0dc0d3f26aac68eca. 2024-11-18T18:27:34,839 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731954260238.fb91ff28ef3c18f0dc0d3f26aac68eca. 2024-11-18T18:27:34,839 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731954260238.fb91ff28ef3c18f0dc0d3f26aac68eca. after waiting 0 ms 2024-11-18T18:27:34,839 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731954260238.fb91ff28ef3c18f0dc0d3f26aac68eca. 2024-11-18T18:27:34,839 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing fb91ff28ef3c18f0dc0d3f26aac68eca 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-18T18:27:34,844 DEBUG [RS_CLOSE_META-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/hbase/meta/1588230740/.tmp/info/09e7f9cb30d941eaae146c840aa7e657 is 268, key is testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731954260238.43138a6a12c2efea7093064622bb3cd8./info:regioninfo/1731954452026/Put/seqid=0 2024-11-18T18:27:34,856 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/75a13ed51395b1016b90c6b0d90a7d0a/.tmp/colfamily11/8589c82d87c74c9d90f7a01bf24ee7ae is 52, key is sss/colfamily11:colfamily11/1731954264675/Put/seqid=0 2024-11-18T18:27:34,857 INFO [regionserver/961a81da9827:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T18:27:34,858 INFO [regionserver/961a81da9827:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T18:27:34,866 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fb91ff28ef3c18f0dc0d3f26aac68eca/.tmp/colfamily11/62b6238347fa40ec837ce6a4e394e624 is 52, key is fff/colfamily11:colfamily11/1731954264331/Put/seqid=0 2024-11-18T18:27:34,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741887_1063 (size=19858) 2024-11-18T18:27:34,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741887_1063 (size=19858) 2024-11-18T18:27:34,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741887_1063 (size=19858) 2024-11-18T18:27:34,891 INFO [RS_CLOSE_META-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.39 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/hbase/meta/1588230740/.tmp/info/09e7f9cb30d941eaae146c840aa7e657 2024-11-18T18:27:34,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741889_1065 (size=39263) 2024-11-18T18:27:34,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741889_1065 (size=39263) 2024-11-18T18:27:34,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741889_1065 (size=39263) 2024-11-18T18:27:34,906 DEBUG [RS_CLOSE_META-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/hbase/meta/1588230740/.tmp/info/09e7f9cb30d941eaae146c840aa7e657 as hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/hbase/meta/1588230740/info/09e7f9cb30d941eaae146c840aa7e657 2024-11-18T18:27:34,907 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fb91ff28ef3c18f0dc0d3f26aac68eca/.tmp/colfamily11/62b6238347fa40ec837ce6a4e394e624 2024-11-18T18:27:34,914 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fb91ff28ef3c18f0dc0d3f26aac68eca/.tmp/colfamily11/62b6238347fa40ec837ce6a4e394e624 as hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fb91ff28ef3c18f0dc0d3f26aac68eca/colfamily11/62b6238347fa40ec837ce6a4e394e624 2024-11-18T18:27:34,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741888_1064 (size=39263) 2024-11-18T18:27:34,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741888_1064 (size=39263) 2024-11-18T18:27:34,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741888_1064 (size=39263) 2024-11-18T18:27:34,920 INFO [RS_CLOSE_META-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/hbase/meta/1588230740/info/09e7f9cb30d941eaae146c840aa7e657, entries=72, sequenceid=81, filesize=19.4 K 2024-11-18T18:27:34,922 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/75a13ed51395b1016b90c6b0d90a7d0a/.tmp/colfamily11/8589c82d87c74c9d90f7a01bf24ee7ae 2024-11-18T18:27:34,925 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fb91ff28ef3c18f0dc0d3f26aac68eca/colfamily11/62b6238347fa40ec837ce6a4e394e624, entries=703, sequenceid=5, filesize=38.3 K 2024-11-18T18:27:34,926 INFO [RS_CLOSE_META-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~13.39 KB/13716, heapSize ~18.94 KB/19392, currentSize=0 B/0 for 1588230740 in 109ms, sequenceid=81, compaction requested=false 2024-11-18T18:27:34,926 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for fb91ff28ef3c18f0dc0d3f26aac68eca in 87ms, sequenceid=5, compaction requested=false 2024-11-18T18:27:34,931 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/75a13ed51395b1016b90c6b0d90a7d0a/.tmp/colfamily11/8589c82d87c74c9d90f7a01bf24ee7ae as hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/75a13ed51395b1016b90c6b0d90a7d0a/colfamily11/8589c82d87c74c9d90f7a01bf24ee7ae 2024-11-18T18:27:34,938 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/75a13ed51395b1016b90c6b0d90a7d0a/colfamily11/8589c82d87c74c9d90f7a01bf24ee7ae, entries=703, sequenceid=5, filesize=38.3 K 2024-11-18T18:27:34,939 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 75a13ed51395b1016b90c6b0d90a7d0a in 128ms, sequenceid=5, compaction requested=false 2024-11-18T18:27:34,949 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fb91ff28ef3c18f0dc0d3f26aac68eca/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-18T18:27:34,950 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731954260238.fb91ff28ef3c18f0dc0d3f26aac68eca. 2024-11-18T18:27:34,950 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fb91ff28ef3c18f0dc0d3f26aac68eca: Waiting for close lock at 1731954454839Running coprocessor pre-close hooks at 1731954454839Disabling compacts and flushes for region at 1731954454839Disabling writes for close at 1731954454839Obtaining lock to block concurrent updates at 1731954454839Preparing flush snapshotting stores in fb91ff28ef3c18f0dc0d3f26aac68eca at 1731954454839Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731954260238.fb91ff28ef3c18f0dc0d3f26aac68eca., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731954454840 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731954260238.fb91ff28ef3c18f0dc0d3f26aac68eca. at 1731954454840Flushing fb91ff28ef3c18f0dc0d3f26aac68eca/colfamily11: creating writer at 1731954454841 (+1 ms)Flushing fb91ff28ef3c18f0dc0d3f26aac68eca/colfamily11: appending metadata at 1731954454866 (+25 ms)Flushing fb91ff28ef3c18f0dc0d3f26aac68eca/colfamily11: closing flushed file at 1731954454866Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6aa38252: reopening flushed file at 1731954454913 (+47 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for fb91ff28ef3c18f0dc0d3f26aac68eca in 87ms, sequenceid=5, compaction requested=false at 1731954454926 (+13 ms)Writing region close event to WAL at 1731954454940 (+14 ms)Running coprocessor post-close hooks at 1731954454950 (+10 ms)Closed at 1731954454950 2024-11-18T18:27:34,950 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731954260238.fb91ff28ef3c18f0dc0d3f26aac68eca. 2024-11-18T18:27:34,951 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 140f5077d3b730b643b558ebe8a13abd, disabling compactions & flushes 2024-11-18T18:27:34,951 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731954260238.140f5077d3b730b643b558ebe8a13abd. 2024-11-18T18:27:34,951 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731954260238.140f5077d3b730b643b558ebe8a13abd. 2024-11-18T18:27:34,951 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731954260238.140f5077d3b730b643b558ebe8a13abd. after waiting 0 ms 2024-11-18T18:27:34,951 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731954260238.140f5077d3b730b643b558ebe8a13abd. 2024-11-18T18:27:34,951 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 140f5077d3b730b643b558ebe8a13abd 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-18T18:27:34,962 DEBUG [RS_CLOSE_META-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/hbase/meta/1588230740/recovered.edits/84.seqid, newMaxSeqId=84, maxSeqId=60 2024-11-18T18:27:34,962 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/75a13ed51395b1016b90c6b0d90a7d0a/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-18T18:27:34,966 DEBUG [RS_CLOSE_META-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-18T18:27:34,966 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731954260238.75a13ed51395b1016b90c6b0d90a7d0a. 2024-11-18T18:27:34,966 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 75a13ed51395b1016b90c6b0d90a7d0a: Waiting for close lock at 1731954454811Running coprocessor pre-close hooks at 1731954454811Disabling compacts and flushes for region at 1731954454811Disabling writes for close at 1731954454811Obtaining lock to block concurrent updates at 1731954454811Preparing flush snapshotting stores in 75a13ed51395b1016b90c6b0d90a7d0a at 1731954454811Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731954260238.75a13ed51395b1016b90c6b0d90a7d0a., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731954454819 (+8 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731954260238.75a13ed51395b1016b90c6b0d90a7d0a. at 1731954454829 (+10 ms)Flushing 75a13ed51395b1016b90c6b0d90a7d0a/colfamily11: creating writer at 1731954454829Flushing 75a13ed51395b1016b90c6b0d90a7d0a/colfamily11: appending metadata at 1731954454856 (+27 ms)Flushing 75a13ed51395b1016b90c6b0d90a7d0a/colfamily11: closing flushed file at 1731954454856Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7702e7af: reopening flushed file at 1731954454930 (+74 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 75a13ed51395b1016b90c6b0d90a7d0a in 128ms, sequenceid=5, compaction requested=false at 1731954454939 (+9 ms)Writing region close event to WAL at 1731954454949 (+10 ms)Running coprocessor post-close hooks at 1731954454966 (+17 ms)Closed at 1731954454966 2024-11-18T18:27:34,966 INFO [RS_CLOSE_META-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-18T18:27:34,966 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731954260238.75a13ed51395b1016b90c6b0d90a7d0a. 2024-11-18T18:27:34,966 DEBUG [RS_CLOSE_META-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731954454816Running coprocessor pre-close hooks at 1731954454816Disabling compacts and flushes for region at 1731954454816Disabling writes for close at 1731954454816Obtaining lock to block concurrent updates at 1731954454817 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731954454817Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=13716, getHeapSize=20112, getOffHeapSize=0, getCellsCount=72 at 1731954454827 (+10 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731954454829 (+2 ms)Flushing 1588230740/info: creating writer at 1731954454830 (+1 ms)Flushing 1588230740/info: appending metadata at 1731954454843 (+13 ms)Flushing 1588230740/info: closing flushed file at 1731954454844 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@760ad401: reopening flushed file at 1731954454904 (+60 ms)Finished flush of dataSize ~13.39 KB/13716, heapSize ~18.94 KB/19392, currentSize=0 B/0 for 1588230740 in 109ms, sequenceid=81, compaction requested=false at 1731954454926 (+22 ms)Writing region close event to WAL at 1731954454950 (+24 ms)Running coprocessor post-close hooks at 1731954454965 (+15 ms)Closed at 1731954454966 (+1 ms) 2024-11-18T18:27:34,966 DEBUG [RS_CLOSE_META-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-18T18:27:34,966 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 012c277c99c564dd90e8e5f6850b6ce0, disabling compactions & flushes 2024-11-18T18:27:34,966 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731954260238.012c277c99c564dd90e8e5f6850b6ce0. 2024-11-18T18:27:34,966 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731954260238.012c277c99c564dd90e8e5f6850b6ce0. 2024-11-18T18:27:34,966 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731954260238.012c277c99c564dd90e8e5f6850b6ce0. after waiting 0 ms 2024-11-18T18:27:34,966 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731954260238.012c277c99c564dd90e8e5f6850b6ce0. 2024-11-18T18:27:34,967 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 012c277c99c564dd90e8e5f6850b6ce0 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-18T18:27:34,980 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/140f5077d3b730b643b558ebe8a13abd/.tmp/colfamily11/e46b063ab9f4460da2860f02d32c8c02 is 52, key is ooo/colfamily11:colfamily11/1731954264542/Put/seqid=0 2024-11-18T18:27:34,991 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/012c277c99c564dd90e8e5f6850b6ce0/.tmp/colfamily11/960e49c882ce49d6a7965751e240bb3c is 52, key is nnn/colfamily11:colfamily11/1731954264564/Put/seqid=0 2024-11-18T18:27:35,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741890_1066 (size=39263) 2024-11-18T18:27:35,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741890_1066 (size=39263) 2024-11-18T18:27:35,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741890_1066 (size=39263) 2024-11-18T18:27:35,009 DEBUG [RS:0;961a81da9827:38709 {}] regionserver.HRegionServer(1351): Waiting on 08ccc32db22fb75f5466ad774ad9e8ba, 140f5077d3b730b643b558ebe8a13abd, 2a8fcaf8905dcde0537806fb16b9ff4c, 5e69ef2147d87d1b83dbcbb4705cbabe, 69ae79c6f9eeed559fa88701f773706c, 8bb7ae0747eda2290799ca8fa83f9ede, 97377a91d35f8b242120f603c456c41e, bcf8695060a326b7d24615ba6cb932b1, c3e2f3fb6516fcaaef4d0a2c8b63842d, f39571a93e25091090ea0831d54936b5 2024-11-18T18:27:35,011 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/140f5077d3b730b643b558ebe8a13abd/.tmp/colfamily11/e46b063ab9f4460da2860f02d32c8c02 2024-11-18T18:27:35,017 DEBUG [RS:2;961a81da9827:42895 {}] regionserver.HRegionServer(1351): Waiting on 012c277c99c564dd90e8e5f6850b6ce0, 22971deb0db38b7cccde5c3bc0868f22, 40a213dc6b2a0fd809c71612c437ab83, 43138a6a12c2efea7093064622bb3cd8, 579ee42a94597fdc1db9a4820a53ee64, 779798cd60db955ed78a4329cac08969, 97a938e093c14701871e85a1fbb6e07d, a244d0ab318eb4dd77bc5235828d9d51, aaf2435418703b2a31c1190af1560ba8, bb099f0975ebdb618ef8e3b2c263bbca, c8b4dd2a6d8dd59085c547b55aac0de6, d1ad74f79ba4d7fd1619c83a55106331, fb8fa4293ea5e58c18dbc5b978355ca9 2024-11-18T18:27:35,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741891_1067 (size=39263) 2024-11-18T18:27:35,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741891_1067 (size=39263) 2024-11-18T18:27:35,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741891_1067 (size=39263) 2024-11-18T18:27:35,025 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/140f5077d3b730b643b558ebe8a13abd/.tmp/colfamily11/e46b063ab9f4460da2860f02d32c8c02 as hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/140f5077d3b730b643b558ebe8a13abd/colfamily11/e46b063ab9f4460da2860f02d32c8c02 2024-11-18T18:27:35,032 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/140f5077d3b730b643b558ebe8a13abd/colfamily11/e46b063ab9f4460da2860f02d32c8c02, entries=703, sequenceid=5, filesize=38.3 K 2024-11-18T18:27:35,033 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 140f5077d3b730b643b558ebe8a13abd in 82ms, sequenceid=5, compaction requested=false 2024-11-18T18:27:35,042 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/140f5077d3b730b643b558ebe8a13abd/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-18T18:27:35,043 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731954260238.140f5077d3b730b643b558ebe8a13abd. 2024-11-18T18:27:35,043 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 140f5077d3b730b643b558ebe8a13abd: Waiting for close lock at 1731954454951Running coprocessor pre-close hooks at 1731954454951Disabling compacts and flushes for region at 1731954454951Disabling writes for close at 1731954454951Obtaining lock to block concurrent updates at 1731954454951Preparing flush snapshotting stores in 140f5077d3b730b643b558ebe8a13abd at 1731954454951Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731954260238.140f5077d3b730b643b558ebe8a13abd., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731954454951Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731954260238.140f5077d3b730b643b558ebe8a13abd. at 1731954454952 (+1 ms)Flushing 140f5077d3b730b643b558ebe8a13abd/colfamily11: creating writer at 1731954454952Flushing 140f5077d3b730b643b558ebe8a13abd/colfamily11: appending metadata at 1731954454979 (+27 ms)Flushing 140f5077d3b730b643b558ebe8a13abd/colfamily11: closing flushed file at 1731954454979Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3eb16cc3: reopening flushed file at 1731954455024 (+45 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 140f5077d3b730b643b558ebe8a13abd in 82ms, sequenceid=5, compaction requested=false at 1731954455033 (+9 ms)Writing region close event to WAL at 1731954455034 (+1 ms)Running coprocessor post-close hooks at 1731954455043 (+9 ms)Closed at 1731954455043 2024-11-18T18:27:35,043 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731954260238.140f5077d3b730b643b558ebe8a13abd. 2024-11-18T18:27:35,043 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8bb7ae0747eda2290799ca8fa83f9ede, disabling compactions & flushes 2024-11-18T18:27:35,043 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731954260238.8bb7ae0747eda2290799ca8fa83f9ede. 2024-11-18T18:27:35,043 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731954260238.8bb7ae0747eda2290799ca8fa83f9ede. 2024-11-18T18:27:35,044 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731954260238.8bb7ae0747eda2290799ca8fa83f9ede. after waiting 0 ms 2024-11-18T18:27:35,044 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731954260238.8bb7ae0747eda2290799ca8fa83f9ede. 2024-11-18T18:27:35,044 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 8bb7ae0747eda2290799ca8fa83f9ede 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-18T18:27:35,072 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8bb7ae0747eda2290799ca8fa83f9ede/.tmp/colfamily11/dc90f17f8c8049729283142f878c5de7 is 52, key is aaa/colfamily11:colfamily11/1731954264228/Put/seqid=0 2024-11-18T18:27:35,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741892_1068 (size=39263) 2024-11-18T18:27:35,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741892_1068 (size=39263) 2024-11-18T18:27:35,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741892_1068 (size=39263) 2024-11-18T18:27:35,094 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8bb7ae0747eda2290799ca8fa83f9ede/.tmp/colfamily11/dc90f17f8c8049729283142f878c5de7 2024-11-18T18:27:35,101 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8bb7ae0747eda2290799ca8fa83f9ede/.tmp/colfamily11/dc90f17f8c8049729283142f878c5de7 as hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8bb7ae0747eda2290799ca8fa83f9ede/colfamily11/dc90f17f8c8049729283142f878c5de7 2024-11-18T18:27:35,109 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8bb7ae0747eda2290799ca8fa83f9ede/colfamily11/dc90f17f8c8049729283142f878c5de7, entries=703, sequenceid=5, filesize=38.3 K 2024-11-18T18:27:35,110 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 8bb7ae0747eda2290799ca8fa83f9ede in 66ms, sequenceid=5, compaction requested=false 2024-11-18T18:27:35,115 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8bb7ae0747eda2290799ca8fa83f9ede/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-18T18:27:35,116 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731954260238.8bb7ae0747eda2290799ca8fa83f9ede. 2024-11-18T18:27:35,116 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8bb7ae0747eda2290799ca8fa83f9ede: Waiting for close lock at 1731954455043Running coprocessor pre-close hooks at 1731954455043Disabling compacts and flushes for region at 1731954455043Disabling writes for close at 1731954455044 (+1 ms)Obtaining lock to block concurrent updates at 1731954455044Preparing flush snapshotting stores in 8bb7ae0747eda2290799ca8fa83f9ede at 1731954455044Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731954260238.8bb7ae0747eda2290799ca8fa83f9ede., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731954455044Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731954260238.8bb7ae0747eda2290799ca8fa83f9ede. at 1731954455045 (+1 ms)Flushing 8bb7ae0747eda2290799ca8fa83f9ede/colfamily11: creating writer at 1731954455045Flushing 8bb7ae0747eda2290799ca8fa83f9ede/colfamily11: appending metadata at 1731954455071 (+26 ms)Flushing 8bb7ae0747eda2290799ca8fa83f9ede/colfamily11: closing flushed file at 1731954455071Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59e6b59d: reopening flushed file at 1731954455100 (+29 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 8bb7ae0747eda2290799ca8fa83f9ede in 66ms, sequenceid=5, compaction requested=false at 1731954455110 (+10 ms)Writing region close event to WAL at 1731954455111 (+1 ms)Running coprocessor post-close hooks at 1731954455115 (+4 ms)Closed at 1731954455115 2024-11-18T18:27:35,116 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731954260238.8bb7ae0747eda2290799ca8fa83f9ede. 2024-11-18T18:27:35,116 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c3e2f3fb6516fcaaef4d0a2c8b63842d, disabling compactions & flushes 2024-11-18T18:27:35,116 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731954260238.c3e2f3fb6516fcaaef4d0a2c8b63842d. 2024-11-18T18:27:35,116 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731954260238.c3e2f3fb6516fcaaef4d0a2c8b63842d. 2024-11-18T18:27:35,116 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731954260238.c3e2f3fb6516fcaaef4d0a2c8b63842d. after waiting 0 ms 2024-11-18T18:27:35,116 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731954260238.c3e2f3fb6516fcaaef4d0a2c8b63842d. 2024-11-18T18:27:35,123 INFO [regionserver/961a81da9827:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-18T18:27:35,123 INFO [regionserver/961a81da9827:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-18T18:27:35,148 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c3e2f3fb6516fcaaef4d0a2c8b63842d/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-18T18:27:35,149 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731954260238.c3e2f3fb6516fcaaef4d0a2c8b63842d. 2024-11-18T18:27:35,149 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c3e2f3fb6516fcaaef4d0a2c8b63842d: Waiting for close lock at 1731954455116Running coprocessor pre-close hooks at 1731954455116Disabling compacts and flushes for region at 1731954455116Disabling writes for close at 1731954455116Writing region close event to WAL at 1731954455136 (+20 ms)Running coprocessor post-close hooks at 1731954455148 (+12 ms)Closed at 1731954455149 (+1 ms) 2024-11-18T18:27:35,149 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731954260238.c3e2f3fb6516fcaaef4d0a2c8b63842d. 2024-11-18T18:27:35,149 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5e69ef2147d87d1b83dbcbb4705cbabe, disabling compactions & flushes 2024-11-18T18:27:35,149 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731954260238.5e69ef2147d87d1b83dbcbb4705cbabe. 2024-11-18T18:27:35,149 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731954260238.5e69ef2147d87d1b83dbcbb4705cbabe. 2024-11-18T18:27:35,149 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731954260238.5e69ef2147d87d1b83dbcbb4705cbabe. after waiting 0 ms 2024-11-18T18:27:35,149 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731954260238.5e69ef2147d87d1b83dbcbb4705cbabe. 2024-11-18T18:27:35,163 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5e69ef2147d87d1b83dbcbb4705cbabe/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-18T18:27:35,164 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731954260238.5e69ef2147d87d1b83dbcbb4705cbabe. 2024-11-18T18:27:35,164 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5e69ef2147d87d1b83dbcbb4705cbabe: Waiting for close lock at 1731954455149Running coprocessor pre-close hooks at 1731954455149Disabling compacts and flushes for region at 1731954455149Disabling writes for close at 1731954455149Writing region close event to WAL at 1731954455150 (+1 ms)Running coprocessor post-close hooks at 1731954455164 (+14 ms)Closed at 1731954455164 2024-11-18T18:27:35,164 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731954260238.5e69ef2147d87d1b83dbcbb4705cbabe. 2024-11-18T18:27:35,165 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2a8fcaf8905dcde0537806fb16b9ff4c, disabling compactions & flushes 2024-11-18T18:27:35,165 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731954260238.2a8fcaf8905dcde0537806fb16b9ff4c. 2024-11-18T18:27:35,165 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731954260238.2a8fcaf8905dcde0537806fb16b9ff4c. 2024-11-18T18:27:35,165 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731954260238.2a8fcaf8905dcde0537806fb16b9ff4c. after waiting 0 ms 2024-11-18T18:27:35,165 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731954260238.2a8fcaf8905dcde0537806fb16b9ff4c. 2024-11-18T18:27:35,165 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 2a8fcaf8905dcde0537806fb16b9ff4c 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-18T18:27:35,193 INFO [regionserver/961a81da9827:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-18T18:27:35,194 INFO [regionserver/961a81da9827:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-18T18:27:35,194 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2a8fcaf8905dcde0537806fb16b9ff4c/.tmp/colfamily11/b177889520634fbdaa896fb496267850 is 52, key is www/colfamily11:colfamily11/1731954264640/Put/seqid=0 2024-11-18T18:27:35,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741893_1069 (size=39263) 2024-11-18T18:27:35,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741893_1069 (size=39263) 2024-11-18T18:27:35,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741893_1069 (size=39263) 2024-11-18T18:27:35,209 DEBUG [RS:0;961a81da9827:38709 {}] regionserver.HRegionServer(1351): Waiting on 08ccc32db22fb75f5466ad774ad9e8ba, 2a8fcaf8905dcde0537806fb16b9ff4c, 69ae79c6f9eeed559fa88701f773706c, 97377a91d35f8b242120f603c456c41e, bcf8695060a326b7d24615ba6cb932b1, f39571a93e25091090ea0831d54936b5 2024-11-18T18:27:35,209 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2a8fcaf8905dcde0537806fb16b9ff4c/.tmp/colfamily11/b177889520634fbdaa896fb496267850 2024-11-18T18:27:35,218 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2a8fcaf8905dcde0537806fb16b9ff4c/.tmp/colfamily11/b177889520634fbdaa896fb496267850 as hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2a8fcaf8905dcde0537806fb16b9ff4c/colfamily11/b177889520634fbdaa896fb496267850 2024-11-18T18:27:35,221 DEBUG [RS:2;961a81da9827:42895 {}] regionserver.HRegionServer(1351): Waiting on 012c277c99c564dd90e8e5f6850b6ce0, 22971deb0db38b7cccde5c3bc0868f22, 40a213dc6b2a0fd809c71612c437ab83, 43138a6a12c2efea7093064622bb3cd8, 579ee42a94597fdc1db9a4820a53ee64, 779798cd60db955ed78a4329cac08969, 97a938e093c14701871e85a1fbb6e07d, a244d0ab318eb4dd77bc5235828d9d51, aaf2435418703b2a31c1190af1560ba8, bb099f0975ebdb618ef8e3b2c263bbca, c8b4dd2a6d8dd59085c547b55aac0de6, d1ad74f79ba4d7fd1619c83a55106331, fb8fa4293ea5e58c18dbc5b978355ca9 2024-11-18T18:27:35,225 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2a8fcaf8905dcde0537806fb16b9ff4c/colfamily11/b177889520634fbdaa896fb496267850, entries=703, sequenceid=5, filesize=38.3 K 2024-11-18T18:27:35,226 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 2a8fcaf8905dcde0537806fb16b9ff4c in 61ms, sequenceid=5, compaction requested=false 2024-11-18T18:27:35,231 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2a8fcaf8905dcde0537806fb16b9ff4c/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-18T18:27:35,232 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731954260238.2a8fcaf8905dcde0537806fb16b9ff4c. 2024-11-18T18:27:35,232 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2a8fcaf8905dcde0537806fb16b9ff4c: Waiting for close lock at 1731954455165Running coprocessor pre-close hooks at 1731954455165Disabling compacts and flushes for region at 1731954455165Disabling writes for close at 1731954455165Obtaining lock to block concurrent updates at 1731954455165Preparing flush snapshotting stores in 2a8fcaf8905dcde0537806fb16b9ff4c at 1731954455165Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731954260238.2a8fcaf8905dcde0537806fb16b9ff4c., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731954455165Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731954260238.2a8fcaf8905dcde0537806fb16b9ff4c. at 1731954455166 (+1 ms)Flushing 2a8fcaf8905dcde0537806fb16b9ff4c/colfamily11: creating writer at 1731954455166Flushing 2a8fcaf8905dcde0537806fb16b9ff4c/colfamily11: appending metadata at 1731954455194 (+28 ms)Flushing 2a8fcaf8905dcde0537806fb16b9ff4c/colfamily11: closing flushed file at 1731954455194Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2588edd: reopening flushed file at 1731954455217 (+23 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 2a8fcaf8905dcde0537806fb16b9ff4c in 61ms, sequenceid=5, compaction requested=false at 1731954455226 (+9 ms)Writing region close event to WAL at 1731954455227 (+1 ms)Running coprocessor post-close hooks at 1731954455232 (+5 ms)Closed at 1731954455232 2024-11-18T18:27:35,232 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731954260238.2a8fcaf8905dcde0537806fb16b9ff4c. 2024-11-18T18:27:35,232 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 08ccc32db22fb75f5466ad774ad9e8ba, disabling compactions & flushes 2024-11-18T18:27:35,232 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731954260238.08ccc32db22fb75f5466ad774ad9e8ba. 2024-11-18T18:27:35,232 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731954260238.08ccc32db22fb75f5466ad774ad9e8ba. 2024-11-18T18:27:35,232 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731954260238.08ccc32db22fb75f5466ad774ad9e8ba. after waiting 0 ms 2024-11-18T18:27:35,232 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731954260238.08ccc32db22fb75f5466ad774ad9e8ba. 2024-11-18T18:27:35,232 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 08ccc32db22fb75f5466ad774ad9e8ba 3/3 column families, dataSize=48 B heapSize=888 B 2024-11-18T18:27:35,252 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/08ccc32db22fb75f5466ad774ad9e8ba/.tmp/colfamily11/4e5f1122741d4040b6cac4380d1e6deb is 52, key is zzz/colfamily11:colfamily11/1731954264648/Put/seqid=0 2024-11-18T18:27:35,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741894_1070 (size=4857) 2024-11-18T18:27:35,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741894_1070 (size=4857) 2024-11-18T18:27:35,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741894_1070 (size=4857) 2024-11-18T18:27:35,409 DEBUG [RS:0;961a81da9827:38709 {}] regionserver.HRegionServer(1351): Waiting on 08ccc32db22fb75f5466ad774ad9e8ba, 69ae79c6f9eeed559fa88701f773706c, 97377a91d35f8b242120f603c456c41e, bcf8695060a326b7d24615ba6cb932b1, f39571a93e25091090ea0831d54936b5 2024-11-18T18:27:35,419 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/012c277c99c564dd90e8e5f6850b6ce0/.tmp/colfamily11/960e49c882ce49d6a7965751e240bb3c 2024-11-18T18:27:35,421 DEBUG [RS:2;961a81da9827:42895 {}] regionserver.HRegionServer(1351): Waiting on 012c277c99c564dd90e8e5f6850b6ce0, 22971deb0db38b7cccde5c3bc0868f22, 40a213dc6b2a0fd809c71612c437ab83, 43138a6a12c2efea7093064622bb3cd8, 579ee42a94597fdc1db9a4820a53ee64, 779798cd60db955ed78a4329cac08969, 97a938e093c14701871e85a1fbb6e07d, a244d0ab318eb4dd77bc5235828d9d51, aaf2435418703b2a31c1190af1560ba8, bb099f0975ebdb618ef8e3b2c263bbca, c8b4dd2a6d8dd59085c547b55aac0de6, d1ad74f79ba4d7fd1619c83a55106331, fb8fa4293ea5e58c18dbc5b978355ca9 2024-11-18T18:27:35,427 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/012c277c99c564dd90e8e5f6850b6ce0/.tmp/colfamily11/960e49c882ce49d6a7965751e240bb3c as hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/012c277c99c564dd90e8e5f6850b6ce0/colfamily11/960e49c882ce49d6a7965751e240bb3c 2024-11-18T18:27:35,434 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/012c277c99c564dd90e8e5f6850b6ce0/colfamily11/960e49c882ce49d6a7965751e240bb3c, entries=703, sequenceid=5, filesize=38.3 K 2024-11-18T18:27:35,436 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 012c277c99c564dd90e8e5f6850b6ce0 in 469ms, sequenceid=5, compaction requested=false 2024-11-18T18:27:35,442 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/012c277c99c564dd90e8e5f6850b6ce0/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-18T18:27:35,442 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731954260238.012c277c99c564dd90e8e5f6850b6ce0. 2024-11-18T18:27:35,443 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 012c277c99c564dd90e8e5f6850b6ce0: Waiting for close lock at 1731954454966Running coprocessor pre-close hooks at 1731954454966Disabling compacts and flushes for region at 1731954454966Disabling writes for close at 1731954454966Obtaining lock to block concurrent updates at 1731954454967 (+1 ms)Preparing flush snapshotting stores in 012c277c99c564dd90e8e5f6850b6ce0 at 1731954454967Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731954260238.012c277c99c564dd90e8e5f6850b6ce0., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731954454967Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731954260238.012c277c99c564dd90e8e5f6850b6ce0. at 1731954454969 (+2 ms)Flushing 012c277c99c564dd90e8e5f6850b6ce0/colfamily11: creating writer at 1731954454969Flushing 012c277c99c564dd90e8e5f6850b6ce0/colfamily11: appending metadata at 1731954454990 (+21 ms)Flushing 012c277c99c564dd90e8e5f6850b6ce0/colfamily11: closing flushed file at 1731954454990Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68c5ffa: reopening flushed file at 1731954455426 (+436 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 012c277c99c564dd90e8e5f6850b6ce0 in 469ms, sequenceid=5, compaction requested=false at 1731954455436 (+10 ms)Writing region close event to WAL at 1731954455437 (+1 ms)Running coprocessor post-close hooks at 1731954455442 (+5 ms)Closed at 1731954455442 2024-11-18T18:27:35,443 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731954260238.012c277c99c564dd90e8e5f6850b6ce0. 2024-11-18T18:27:35,443 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 22971deb0db38b7cccde5c3bc0868f22, disabling compactions & flushes 2024-11-18T18:27:35,443 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731954260238.22971deb0db38b7cccde5c3bc0868f22. 2024-11-18T18:27:35,443 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731954260238.22971deb0db38b7cccde5c3bc0868f22. 2024-11-18T18:27:35,443 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731954260238.22971deb0db38b7cccde5c3bc0868f22. after waiting 0 ms 2024-11-18T18:27:35,443 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731954260238.22971deb0db38b7cccde5c3bc0868f22. 2024-11-18T18:27:35,443 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 22971deb0db38b7cccde5c3bc0868f22 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-18T18:27:35,462 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/22971deb0db38b7cccde5c3bc0868f22/.tmp/colfamily11/315d9bb6cca741e9b2a90da6e84655e3 is 52, key is bbb/colfamily11:colfamily11/1731954264245/Put/seqid=0 2024-11-18T18:27:35,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741895_1071 (size=39263) 2024-11-18T18:27:35,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741895_1071 (size=39263) 2024-11-18T18:27:35,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741895_1071 (size=39263) 2024-11-18T18:27:35,472 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/22971deb0db38b7cccde5c3bc0868f22/.tmp/colfamily11/315d9bb6cca741e9b2a90da6e84655e3 2024-11-18T18:27:35,479 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/22971deb0db38b7cccde5c3bc0868f22/.tmp/colfamily11/315d9bb6cca741e9b2a90da6e84655e3 as hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/22971deb0db38b7cccde5c3bc0868f22/colfamily11/315d9bb6cca741e9b2a90da6e84655e3 2024-11-18T18:27:35,490 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/22971deb0db38b7cccde5c3bc0868f22/colfamily11/315d9bb6cca741e9b2a90da6e84655e3, entries=703, sequenceid=5, filesize=38.3 K 2024-11-18T18:27:35,492 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 22971deb0db38b7cccde5c3bc0868f22 in 48ms, sequenceid=5, compaction requested=false 2024-11-18T18:27:35,497 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/22971deb0db38b7cccde5c3bc0868f22/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-18T18:27:35,498 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731954260238.22971deb0db38b7cccde5c3bc0868f22. 2024-11-18T18:27:35,498 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 22971deb0db38b7cccde5c3bc0868f22: Waiting for close lock at 1731954455443Running coprocessor pre-close hooks at 1731954455443Disabling compacts and flushes for region at 1731954455443Disabling writes for close at 1731954455443Obtaining lock to block concurrent updates at 1731954455443Preparing flush snapshotting stores in 22971deb0db38b7cccde5c3bc0868f22 at 1731954455443Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731954260238.22971deb0db38b7cccde5c3bc0868f22., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731954455443Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731954260238.22971deb0db38b7cccde5c3bc0868f22. at 1731954455444 (+1 ms)Flushing 22971deb0db38b7cccde5c3bc0868f22/colfamily11: creating writer at 1731954455444Flushing 22971deb0db38b7cccde5c3bc0868f22/colfamily11: appending metadata at 1731954455462 (+18 ms)Flushing 22971deb0db38b7cccde5c3bc0868f22/colfamily11: closing flushed file at 1731954455462Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@45bed217: reopening flushed file at 1731954455478 (+16 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 22971deb0db38b7cccde5c3bc0868f22 in 48ms, sequenceid=5, compaction requested=false at 1731954455492 (+14 ms)Writing region close event to WAL at 1731954455493 (+1 ms)Running coprocessor post-close hooks at 1731954455498 (+5 ms)Closed at 1731954455498 2024-11-18T18:27:35,498 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731954260238.22971deb0db38b7cccde5c3bc0868f22. 2024-11-18T18:27:35,498 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 579ee42a94597fdc1db9a4820a53ee64, disabling compactions & flushes 2024-11-18T18:27:35,498 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731954260238.579ee42a94597fdc1db9a4820a53ee64. 2024-11-18T18:27:35,498 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731954260238.579ee42a94597fdc1db9a4820a53ee64. 2024-11-18T18:27:35,499 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731954260238.579ee42a94597fdc1db9a4820a53ee64. after waiting 0 ms 2024-11-18T18:27:35,499 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731954260238.579ee42a94597fdc1db9a4820a53ee64. 2024-11-18T18:27:35,499 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 579ee42a94597fdc1db9a4820a53ee64 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-18T18:27:35,526 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/579ee42a94597fdc1db9a4820a53ee64/.tmp/colfamily11/bfa6d9e9b6c1421686e417195c1e3df5 is 52, key is xxx/colfamily11:colfamily11/1731954264755/Put/seqid=0 2024-11-18T18:27:35,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741896_1072 (size=39263) 2024-11-18T18:27:35,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741896_1072 (size=39263) 2024-11-18T18:27:35,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741896_1072 (size=39263) 2024-11-18T18:27:35,545 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/579ee42a94597fdc1db9a4820a53ee64/.tmp/colfamily11/bfa6d9e9b6c1421686e417195c1e3df5 2024-11-18T18:27:35,555 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/579ee42a94597fdc1db9a4820a53ee64/.tmp/colfamily11/bfa6d9e9b6c1421686e417195c1e3df5 as hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/579ee42a94597fdc1db9a4820a53ee64/colfamily11/bfa6d9e9b6c1421686e417195c1e3df5 2024-11-18T18:27:35,566 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/579ee42a94597fdc1db9a4820a53ee64/colfamily11/bfa6d9e9b6c1421686e417195c1e3df5, entries=703, sequenceid=5, filesize=38.3 K 2024-11-18T18:27:35,567 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 579ee42a94597fdc1db9a4820a53ee64 in 68ms, sequenceid=5, compaction requested=false 2024-11-18T18:27:35,572 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/579ee42a94597fdc1db9a4820a53ee64/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-18T18:27:35,573 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731954260238.579ee42a94597fdc1db9a4820a53ee64. 2024-11-18T18:27:35,573 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 579ee42a94597fdc1db9a4820a53ee64: Waiting for close lock at 1731954455498Running coprocessor pre-close hooks at 1731954455498Disabling compacts and flushes for region at 1731954455498Disabling writes for close at 1731954455499 (+1 ms)Obtaining lock to block concurrent updates at 1731954455499Preparing flush snapshotting stores in 579ee42a94597fdc1db9a4820a53ee64 at 1731954455499Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731954260238.579ee42a94597fdc1db9a4820a53ee64., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731954455499Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731954260238.579ee42a94597fdc1db9a4820a53ee64. at 1731954455500 (+1 ms)Flushing 579ee42a94597fdc1db9a4820a53ee64/colfamily11: creating writer at 1731954455500Flushing 579ee42a94597fdc1db9a4820a53ee64/colfamily11: appending metadata at 1731954455525 (+25 ms)Flushing 579ee42a94597fdc1db9a4820a53ee64/colfamily11: closing flushed file at 1731954455525Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33106837: reopening flushed file at 1731954455554 (+29 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 579ee42a94597fdc1db9a4820a53ee64 in 68ms, sequenceid=5, compaction requested=false at 1731954455567 (+13 ms)Writing region close event to WAL at 1731954455568 (+1 ms)Running coprocessor post-close hooks at 1731954455573 (+5 ms)Closed at 1731954455573 2024-11-18T18:27:35,573 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731954260238.579ee42a94597fdc1db9a4820a53ee64. 2024-11-18T18:27:35,573 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 40a213dc6b2a0fd809c71612c437ab83, disabling compactions & flushes 2024-11-18T18:27:35,574 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731954260238.40a213dc6b2a0fd809c71612c437ab83. 2024-11-18T18:27:35,574 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731954260238.40a213dc6b2a0fd809c71612c437ab83. 2024-11-18T18:27:35,574 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731954260238.40a213dc6b2a0fd809c71612c437ab83. after waiting 0 ms 2024-11-18T18:27:35,574 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731954260238.40a213dc6b2a0fd809c71612c437ab83. 2024-11-18T18:27:35,577 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/40a213dc6b2a0fd809c71612c437ab83/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-18T18:27:35,578 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731954260238.40a213dc6b2a0fd809c71612c437ab83. 2024-11-18T18:27:35,578 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 40a213dc6b2a0fd809c71612c437ab83: Waiting for close lock at 1731954455573Running coprocessor pre-close hooks at 1731954455573Disabling compacts and flushes for region at 1731954455573Disabling writes for close at 1731954455574 (+1 ms)Writing region close event to WAL at 1731954455574Running coprocessor post-close hooks at 1731954455578 (+4 ms)Closed at 1731954455578 2024-11-18T18:27:35,578 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731954260238.40a213dc6b2a0fd809c71612c437ab83. 2024-11-18T18:27:35,578 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 779798cd60db955ed78a4329cac08969, disabling compactions & flushes 2024-11-18T18:27:35,578 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731954260238.779798cd60db955ed78a4329cac08969. 2024-11-18T18:27:35,578 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731954260238.779798cd60db955ed78a4329cac08969. 2024-11-18T18:27:35,578 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731954260238.779798cd60db955ed78a4329cac08969. after waiting 0 ms 2024-11-18T18:27:35,578 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731954260238.779798cd60db955ed78a4329cac08969. 2024-11-18T18:27:35,582 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/779798cd60db955ed78a4329cac08969/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-18T18:27:35,583 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731954260238.779798cd60db955ed78a4329cac08969. 2024-11-18T18:27:35,583 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 779798cd60db955ed78a4329cac08969: Waiting for close lock at 1731954455578Running coprocessor pre-close hooks at 1731954455578Disabling compacts and flushes for region at 1731954455578Disabling writes for close at 1731954455578Writing region close event to WAL at 1731954455579 (+1 ms)Running coprocessor post-close hooks at 1731954455583 (+4 ms)Closed at 1731954455583 2024-11-18T18:27:35,583 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731954260238.779798cd60db955ed78a4329cac08969. 2024-11-18T18:27:35,583 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a244d0ab318eb4dd77bc5235828d9d51, disabling compactions & flushes 2024-11-18T18:27:35,583 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731954260238.a244d0ab318eb4dd77bc5235828d9d51. 2024-11-18T18:27:35,583 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731954260238.a244d0ab318eb4dd77bc5235828d9d51. 2024-11-18T18:27:35,583 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731954260238.a244d0ab318eb4dd77bc5235828d9d51. after waiting 0 ms 2024-11-18T18:27:35,583 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731954260238.a244d0ab318eb4dd77bc5235828d9d51. 2024-11-18T18:27:35,587 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a244d0ab318eb4dd77bc5235828d9d51/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-18T18:27:35,587 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731954260238.a244d0ab318eb4dd77bc5235828d9d51. 2024-11-18T18:27:35,587 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a244d0ab318eb4dd77bc5235828d9d51: Waiting for close lock at 1731954455583Running coprocessor pre-close hooks at 1731954455583Disabling compacts and flushes for region at 1731954455583Disabling writes for close at 1731954455583Writing region close event to WAL at 1731954455584 (+1 ms)Running coprocessor post-close hooks at 1731954455587 (+3 ms)Closed at 1731954455587 2024-11-18T18:27:35,587 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731954260238.a244d0ab318eb4dd77bc5235828d9d51. 2024-11-18T18:27:35,587 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing bb099f0975ebdb618ef8e3b2c263bbca, disabling compactions & flushes 2024-11-18T18:27:35,587 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731954260238.bb099f0975ebdb618ef8e3b2c263bbca. 2024-11-18T18:27:35,587 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731954260238.bb099f0975ebdb618ef8e3b2c263bbca. 2024-11-18T18:27:35,588 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731954260238.bb099f0975ebdb618ef8e3b2c263bbca. after waiting 0 ms 2024-11-18T18:27:35,588 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731954260238.bb099f0975ebdb618ef8e3b2c263bbca. 2024-11-18T18:27:35,588 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing bb099f0975ebdb618ef8e3b2c263bbca 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-18T18:27:35,604 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bb099f0975ebdb618ef8e3b2c263bbca/.tmp/colfamily11/01203a2a14274009a6f108c7b8505aff is 52, key is ddd/colfamily11:colfamily11/1731954264322/Put/seqid=0 2024-11-18T18:27:35,609 DEBUG [RS:0;961a81da9827:38709 {}] regionserver.HRegionServer(1351): Waiting on 08ccc32db22fb75f5466ad774ad9e8ba, 69ae79c6f9eeed559fa88701f773706c, 97377a91d35f8b242120f603c456c41e, bcf8695060a326b7d24615ba6cb932b1, f39571a93e25091090ea0831d54936b5 2024-11-18T18:27:35,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741897_1073 (size=39263) 2024-11-18T18:27:35,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741897_1073 (size=39263) 2024-11-18T18:27:35,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741897_1073 (size=39263) 2024-11-18T18:27:35,613 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bb099f0975ebdb618ef8e3b2c263bbca/.tmp/colfamily11/01203a2a14274009a6f108c7b8505aff 2024-11-18T18:27:35,621 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bb099f0975ebdb618ef8e3b2c263bbca/.tmp/colfamily11/01203a2a14274009a6f108c7b8505aff as hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bb099f0975ebdb618ef8e3b2c263bbca/colfamily11/01203a2a14274009a6f108c7b8505aff 2024-11-18T18:27:35,622 DEBUG [RS:2;961a81da9827:42895 {}] regionserver.HRegionServer(1351): Waiting on 43138a6a12c2efea7093064622bb3cd8, 97a938e093c14701871e85a1fbb6e07d, aaf2435418703b2a31c1190af1560ba8, bb099f0975ebdb618ef8e3b2c263bbca, c8b4dd2a6d8dd59085c547b55aac0de6, d1ad74f79ba4d7fd1619c83a55106331, fb8fa4293ea5e58c18dbc5b978355ca9 2024-11-18T18:27:35,634 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bb099f0975ebdb618ef8e3b2c263bbca/colfamily11/01203a2a14274009a6f108c7b8505aff, entries=703, sequenceid=5, filesize=38.3 K 2024-11-18T18:27:35,642 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for bb099f0975ebdb618ef8e3b2c263bbca in 53ms, sequenceid=5, compaction requested=false 2024-11-18T18:27:35,652 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bb099f0975ebdb618ef8e3b2c263bbca/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-18T18:27:35,653 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731954260238.bb099f0975ebdb618ef8e3b2c263bbca. 2024-11-18T18:27:35,653 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for bb099f0975ebdb618ef8e3b2c263bbca: Waiting for close lock at 1731954455587Running coprocessor pre-close hooks at 1731954455587Disabling compacts and flushes for region at 1731954455587Disabling writes for close at 1731954455588 (+1 ms)Obtaining lock to block concurrent updates at 1731954455588Preparing flush snapshotting stores in bb099f0975ebdb618ef8e3b2c263bbca at 1731954455588Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731954260238.bb099f0975ebdb618ef8e3b2c263bbca., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731954455588Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731954260238.bb099f0975ebdb618ef8e3b2c263bbca. at 1731954455588Flushing bb099f0975ebdb618ef8e3b2c263bbca/colfamily11: creating writer at 1731954455588Flushing bb099f0975ebdb618ef8e3b2c263bbca/colfamily11: appending metadata at 1731954455604 (+16 ms)Flushing bb099f0975ebdb618ef8e3b2c263bbca/colfamily11: closing flushed file at 1731954455604Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d46c7e2: reopening flushed file at 1731954455620 (+16 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for bb099f0975ebdb618ef8e3b2c263bbca in 53ms, sequenceid=5, compaction requested=false at 1731954455642 (+22 ms)Writing region close event to WAL at 1731954455647 (+5 ms)Running coprocessor post-close hooks at 1731954455653 (+6 ms)Closed at 1731954455653 2024-11-18T18:27:35,654 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731954260238.bb099f0975ebdb618ef8e3b2c263bbca. 2024-11-18T18:27:35,654 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d1ad74f79ba4d7fd1619c83a55106331, disabling compactions & flushes 2024-11-18T18:27:35,654 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731954260238.d1ad74f79ba4d7fd1619c83a55106331. 2024-11-18T18:27:35,654 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731954260238.d1ad74f79ba4d7fd1619c83a55106331. 2024-11-18T18:27:35,654 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731954260238.d1ad74f79ba4d7fd1619c83a55106331. after waiting 0 ms 2024-11-18T18:27:35,654 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731954260238.d1ad74f79ba4d7fd1619c83a55106331. 2024-11-18T18:27:35,665 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d1ad74f79ba4d7fd1619c83a55106331/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-18T18:27:35,666 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731954260238.d1ad74f79ba4d7fd1619c83a55106331. 2024-11-18T18:27:35,666 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d1ad74f79ba4d7fd1619c83a55106331: Waiting for close lock at 1731954455654Running coprocessor pre-close hooks at 1731954455654Disabling compacts and flushes for region at 1731954455654Disabling writes for close at 1731954455654Writing region close event to WAL at 1731954455655 (+1 ms)Running coprocessor post-close hooks at 1731954455666 (+11 ms)Closed at 1731954455666 2024-11-18T18:27:35,666 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731954260238.d1ad74f79ba4d7fd1619c83a55106331. 2024-11-18T18:27:35,666 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c8b4dd2a6d8dd59085c547b55aac0de6, disabling compactions & flushes 2024-11-18T18:27:35,667 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731954260238.c8b4dd2a6d8dd59085c547b55aac0de6. 2024-11-18T18:27:35,667 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731954260238.c8b4dd2a6d8dd59085c547b55aac0de6. 2024-11-18T18:27:35,667 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731954260238.c8b4dd2a6d8dd59085c547b55aac0de6. after waiting 0 ms 2024-11-18T18:27:35,667 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731954260238.c8b4dd2a6d8dd59085c547b55aac0de6. 2024-11-18T18:27:35,667 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing c8b4dd2a6d8dd59085c547b55aac0de6 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-18T18:27:35,682 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/08ccc32db22fb75f5466ad774ad9e8ba/.tmp/colfamily11/4e5f1122741d4040b6cac4380d1e6deb 2024-11-18T18:27:35,695 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c8b4dd2a6d8dd59085c547b55aac0de6/.tmp/colfamily11/85195b1b991a4688a9bb83b314101c87 is 52, key is ggg/colfamily11:colfamily11/1731954264444/Put/seqid=0 2024-11-18T18:27:35,701 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/08ccc32db22fb75f5466ad774ad9e8ba/.tmp/colfamily11/4e5f1122741d4040b6cac4380d1e6deb as hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/08ccc32db22fb75f5466ad774ad9e8ba/colfamily11/4e5f1122741d4040b6cac4380d1e6deb 2024-11-18T18:27:35,714 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/08ccc32db22fb75f5466ad774ad9e8ba/colfamily11/4e5f1122741d4040b6cac4380d1e6deb, entries=1, sequenceid=5, filesize=4.7 K 2024-11-18T18:27:35,720 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~48 B/48, heapSize ~360 B/360, currentSize=0 B/0 for 08ccc32db22fb75f5466ad774ad9e8ba in 488ms, sequenceid=5, compaction requested=false 2024-11-18T18:27:35,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741898_1074 (size=39263) 2024-11-18T18:27:35,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741898_1074 (size=39263) 2024-11-18T18:27:35,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741898_1074 (size=39263) 2024-11-18T18:27:35,730 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c8b4dd2a6d8dd59085c547b55aac0de6/.tmp/colfamily11/85195b1b991a4688a9bb83b314101c87 2024-11-18T18:27:35,730 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/08ccc32db22fb75f5466ad774ad9e8ba/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-18T18:27:35,731 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731954260238.08ccc32db22fb75f5466ad774ad9e8ba. 2024-11-18T18:27:35,731 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 08ccc32db22fb75f5466ad774ad9e8ba: Waiting for close lock at 1731954455232Running coprocessor pre-close hooks at 1731954455232Disabling compacts and flushes for region at 1731954455232Disabling writes for close at 1731954455232Obtaining lock to block concurrent updates at 1731954455232Preparing flush snapshotting stores in 08ccc32db22fb75f5466ad774ad9e8ba at 1731954455232Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731954260238.08ccc32db22fb75f5466ad774ad9e8ba., syncing WAL and waiting on mvcc, flushsize=dataSize=48, getHeapSize=840, getOffHeapSize=0, getCellsCount=1 at 1731954455233 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731954260238.08ccc32db22fb75f5466ad774ad9e8ba. at 1731954455233Flushing 08ccc32db22fb75f5466ad774ad9e8ba/colfamily11: creating writer at 1731954455233Flushing 08ccc32db22fb75f5466ad774ad9e8ba/colfamily11: appending metadata at 1731954455252 (+19 ms)Flushing 08ccc32db22fb75f5466ad774ad9e8ba/colfamily11: closing flushed file at 1731954455252Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4fbe74dc: reopening flushed file at 1731954455700 (+448 ms)Finished flush of dataSize ~48 B/48, heapSize ~360 B/360, currentSize=0 B/0 for 08ccc32db22fb75f5466ad774ad9e8ba in 488ms, sequenceid=5, compaction requested=false at 1731954455720 (+20 ms)Writing region close event to WAL at 1731954455725 (+5 ms)Running coprocessor post-close hooks at 1731954455731 (+6 ms)Closed at 1731954455731 2024-11-18T18:27:35,731 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731954260238.08ccc32db22fb75f5466ad774ad9e8ba. 2024-11-18T18:27:35,731 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 97377a91d35f8b242120f603c456c41e, disabling compactions & flushes 2024-11-18T18:27:35,731 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731954260238.97377a91d35f8b242120f603c456c41e. 2024-11-18T18:27:35,732 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731954260238.97377a91d35f8b242120f603c456c41e. 2024-11-18T18:27:35,732 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731954260238.97377a91d35f8b242120f603c456c41e. after waiting 0 ms 2024-11-18T18:27:35,732 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731954260238.97377a91d35f8b242120f603c456c41e. 2024-11-18T18:27:35,732 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 97377a91d35f8b242120f603c456c41e 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-18T18:27:35,759 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c8b4dd2a6d8dd59085c547b55aac0de6/.tmp/colfamily11/85195b1b991a4688a9bb83b314101c87 as hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c8b4dd2a6d8dd59085c547b55aac0de6/colfamily11/85195b1b991a4688a9bb83b314101c87 2024-11-18T18:27:35,773 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c8b4dd2a6d8dd59085c547b55aac0de6/colfamily11/85195b1b991a4688a9bb83b314101c87, entries=703, sequenceid=5, filesize=38.3 K 2024-11-18T18:27:35,778 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/97377a91d35f8b242120f603c456c41e/.tmp/colfamily11/496d6f906b5941969aca7140b7c239f4 is 52, key is lll/colfamily11:colfamily11/1731954264496/Put/seqid=0 2024-11-18T18:27:35,780 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for c8b4dd2a6d8dd59085c547b55aac0de6 in 112ms, sequenceid=5, compaction requested=false 2024-11-18T18:27:35,790 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c8b4dd2a6d8dd59085c547b55aac0de6/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-18T18:27:35,791 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731954260238.c8b4dd2a6d8dd59085c547b55aac0de6. 2024-11-18T18:27:35,791 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c8b4dd2a6d8dd59085c547b55aac0de6: Waiting for close lock at 1731954455666Running coprocessor pre-close hooks at 1731954455666Disabling compacts and flushes for region at 1731954455666Disabling writes for close at 1731954455667 (+1 ms)Obtaining lock to block concurrent updates at 1731954455667Preparing flush snapshotting stores in c8b4dd2a6d8dd59085c547b55aac0de6 at 1731954455667Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731954260238.c8b4dd2a6d8dd59085c547b55aac0de6., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731954455667Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731954260238.c8b4dd2a6d8dd59085c547b55aac0de6. at 1731954455669 (+2 ms)Flushing c8b4dd2a6d8dd59085c547b55aac0de6/colfamily11: creating writer at 1731954455669Flushing c8b4dd2a6d8dd59085c547b55aac0de6/colfamily11: appending metadata at 1731954455695 (+26 ms)Flushing c8b4dd2a6d8dd59085c547b55aac0de6/colfamily11: closing flushed file at 1731954455695Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4572dcca: reopening flushed file at 1731954455757 (+62 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for c8b4dd2a6d8dd59085c547b55aac0de6 in 112ms, sequenceid=5, compaction requested=false at 1731954455780 (+23 ms)Writing region close event to WAL at 1731954455782 (+2 ms)Running coprocessor post-close hooks at 1731954455791 (+9 ms)Closed at 1731954455791 2024-11-18T18:27:35,791 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731954260238.c8b4dd2a6d8dd59085c547b55aac0de6. 2024-11-18T18:27:35,792 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fb8fa4293ea5e58c18dbc5b978355ca9, disabling compactions & flushes 2024-11-18T18:27:35,792 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731954260238.fb8fa4293ea5e58c18dbc5b978355ca9. 2024-11-18T18:27:35,792 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731954260238.fb8fa4293ea5e58c18dbc5b978355ca9. 2024-11-18T18:27:35,792 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731954260238.fb8fa4293ea5e58c18dbc5b978355ca9. after waiting 0 ms 2024-11-18T18:27:35,792 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731954260238.fb8fa4293ea5e58c18dbc5b978355ca9. 2024-11-18T18:27:35,792 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing fb8fa4293ea5e58c18dbc5b978355ca9 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-18T18:27:35,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741899_1075 (size=39263) 2024-11-18T18:27:35,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741899_1075 (size=39263) 2024-11-18T18:27:35,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741899_1075 (size=39263) 2024-11-18T18:27:35,812 INFO [RS:0;961a81da9827:38709 {}] regionserver.HRegionServer(1321): Waiting on 4 regions to close 2024-11-18T18:27:35,812 DEBUG [RS:0;961a81da9827:38709 {}] regionserver.HRegionServer(1325): Online Regions={97377a91d35f8b242120f603c456c41e=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731954260238.97377a91d35f8b242120f603c456c41e., 69ae79c6f9eeed559fa88701f773706c=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731954260238.69ae79c6f9eeed559fa88701f773706c., bcf8695060a326b7d24615ba6cb932b1=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731954260238.bcf8695060a326b7d24615ba6cb932b1., f39571a93e25091090ea0831d54936b5=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731954260238.f39571a93e25091090ea0831d54936b5.} 2024-11-18T18:27:35,812 DEBUG [RS:0;961a81da9827:38709 {}] regionserver.HRegionServer(1351): Waiting on 69ae79c6f9eeed559fa88701f773706c, 97377a91d35f8b242120f603c456c41e, bcf8695060a326b7d24615ba6cb932b1, f39571a93e25091090ea0831d54936b5 2024-11-18T18:27:35,812 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/97377a91d35f8b242120f603c456c41e/.tmp/colfamily11/496d6f906b5941969aca7140b7c239f4 2024-11-18T18:27:35,822 INFO [RS:2;961a81da9827:42895 {}] regionserver.HRegionServer(1321): Waiting on 4 regions to close 2024-11-18T18:27:35,822 DEBUG [RS:2;961a81da9827:42895 {}] regionserver.HRegionServer(1325): Online Regions={fb8fa4293ea5e58c18dbc5b978355ca9=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731954260238.fb8fa4293ea5e58c18dbc5b978355ca9., 97a938e093c14701871e85a1fbb6e07d=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731954260238.97a938e093c14701871e85a1fbb6e07d., 43138a6a12c2efea7093064622bb3cd8=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731954260238.43138a6a12c2efea7093064622bb3cd8., aaf2435418703b2a31c1190af1560ba8=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731954260238.aaf2435418703b2a31c1190af1560ba8.} 2024-11-18T18:27:35,822 DEBUG [RS:2;961a81da9827:42895 {}] regionserver.HRegionServer(1351): Waiting on 43138a6a12c2efea7093064622bb3cd8, 97a938e093c14701871e85a1fbb6e07d, aaf2435418703b2a31c1190af1560ba8, fb8fa4293ea5e58c18dbc5b978355ca9 2024-11-18T18:27:35,825 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/97377a91d35f8b242120f603c456c41e/.tmp/colfamily11/496d6f906b5941969aca7140b7c239f4 as hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/97377a91d35f8b242120f603c456c41e/colfamily11/496d6f906b5941969aca7140b7c239f4 2024-11-18T18:27:35,828 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fb8fa4293ea5e58c18dbc5b978355ca9/.tmp/colfamily11/fc8927cd2a2642afba5ca3b936d54730 is 52, key is qqq/colfamily11:colfamily11/1731954264622/Put/seqid=0 2024-11-18T18:27:35,846 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/97377a91d35f8b242120f603c456c41e/colfamily11/496d6f906b5941969aca7140b7c239f4, entries=703, sequenceid=5, filesize=38.3 K 2024-11-18T18:27:35,848 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 97377a91d35f8b242120f603c456c41e in 116ms, sequenceid=5, compaction requested=false 2024-11-18T18:27:35,887 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/97377a91d35f8b242120f603c456c41e/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-18T18:27:35,888 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731954260238.97377a91d35f8b242120f603c456c41e. 2024-11-18T18:27:35,888 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 97377a91d35f8b242120f603c456c41e: Waiting for close lock at 1731954455731Running coprocessor pre-close hooks at 1731954455731Disabling compacts and flushes for region at 1731954455731Disabling writes for close at 1731954455732 (+1 ms)Obtaining lock to block concurrent updates at 1731954455732Preparing flush snapshotting stores in 97377a91d35f8b242120f603c456c41e at 1731954455732Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731954260238.97377a91d35f8b242120f603c456c41e., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731954455732Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731954260238.97377a91d35f8b242120f603c456c41e. at 1731954455733 (+1 ms)Flushing 97377a91d35f8b242120f603c456c41e/colfamily11: creating writer at 1731954455733Flushing 97377a91d35f8b242120f603c456c41e/colfamily11: appending metadata at 1731954455778 (+45 ms)Flushing 97377a91d35f8b242120f603c456c41e/colfamily11: closing flushed file at 1731954455778Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@28d1473e: reopening flushed file at 1731954455824 (+46 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 97377a91d35f8b242120f603c456c41e in 116ms, sequenceid=5, compaction requested=false at 1731954455848 (+24 ms)Writing region close event to WAL at 1731954455875 (+27 ms)Running coprocessor post-close hooks at 1731954455888 (+13 ms)Closed at 1731954455888 2024-11-18T18:27:35,889 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731954260238.97377a91d35f8b242120f603c456c41e. 2024-11-18T18:27:35,889 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 69ae79c6f9eeed559fa88701f773706c, disabling compactions & flushes 2024-11-18T18:27:35,889 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731954260238.69ae79c6f9eeed559fa88701f773706c. 2024-11-18T18:27:35,889 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731954260238.69ae79c6f9eeed559fa88701f773706c. 2024-11-18T18:27:35,889 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731954260238.69ae79c6f9eeed559fa88701f773706c. after waiting 0 ms 2024-11-18T18:27:35,889 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731954260238.69ae79c6f9eeed559fa88701f773706c. 2024-11-18T18:27:35,889 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 69ae79c6f9eeed559fa88701f773706c 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-18T18:27:35,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741900_1076 (size=39263) 2024-11-18T18:27:35,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741900_1076 (size=39263) 2024-11-18T18:27:35,916 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/69ae79c6f9eeed559fa88701f773706c/.tmp/colfamily11/44975fa8c3d443418c8ec841193fee78 is 52, key is rrr/colfamily11:colfamily11/1731954264592/Put/seqid=0 2024-11-18T18:27:35,918 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fb8fa4293ea5e58c18dbc5b978355ca9/.tmp/colfamily11/fc8927cd2a2642afba5ca3b936d54730 2024-11-18T18:27:35,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741900_1076 (size=39263) 2024-11-18T18:27:35,930 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fb8fa4293ea5e58c18dbc5b978355ca9/.tmp/colfamily11/fc8927cd2a2642afba5ca3b936d54730 as hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fb8fa4293ea5e58c18dbc5b978355ca9/colfamily11/fc8927cd2a2642afba5ca3b936d54730 2024-11-18T18:27:35,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741901_1077 (size=39263) 2024-11-18T18:27:35,963 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fb8fa4293ea5e58c18dbc5b978355ca9/colfamily11/fc8927cd2a2642afba5ca3b936d54730, entries=703, sequenceid=5, filesize=38.3 K 2024-11-18T18:27:35,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741901_1077 (size=39263) 2024-11-18T18:27:35,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741901_1077 (size=39263) 2024-11-18T18:27:35,978 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/69ae79c6f9eeed559fa88701f773706c/.tmp/colfamily11/44975fa8c3d443418c8ec841193fee78 2024-11-18T18:27:35,979 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for fb8fa4293ea5e58c18dbc5b978355ca9 in 187ms, sequenceid=5, compaction requested=false 2024-11-18T18:27:36,009 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/69ae79c6f9eeed559fa88701f773706c/.tmp/colfamily11/44975fa8c3d443418c8ec841193fee78 as hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/69ae79c6f9eeed559fa88701f773706c/colfamily11/44975fa8c3d443418c8ec841193fee78 2024-11-18T18:27:36,013 DEBUG [RS:0;961a81da9827:38709 {}] regionserver.HRegionServer(1351): Waiting on 69ae79c6f9eeed559fa88701f773706c, bcf8695060a326b7d24615ba6cb932b1, f39571a93e25091090ea0831d54936b5 2024-11-18T18:27:36,020 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fb8fa4293ea5e58c18dbc5b978355ca9/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-18T18:27:36,021 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731954260238.fb8fa4293ea5e58c18dbc5b978355ca9. 2024-11-18T18:27:36,021 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fb8fa4293ea5e58c18dbc5b978355ca9: Waiting for close lock at 1731954455792Running coprocessor pre-close hooks at 1731954455792Disabling compacts and flushes for region at 1731954455792Disabling writes for close at 1731954455792Obtaining lock to block concurrent updates at 1731954455792Preparing flush snapshotting stores in fb8fa4293ea5e58c18dbc5b978355ca9 at 1731954455792Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731954260238.fb8fa4293ea5e58c18dbc5b978355ca9., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731954455792Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731954260238.fb8fa4293ea5e58c18dbc5b978355ca9. at 1731954455797 (+5 ms)Flushing fb8fa4293ea5e58c18dbc5b978355ca9/colfamily11: creating writer at 1731954455798 (+1 ms)Flushing fb8fa4293ea5e58c18dbc5b978355ca9/colfamily11: appending metadata at 1731954455828 (+30 ms)Flushing fb8fa4293ea5e58c18dbc5b978355ca9/colfamily11: closing flushed file at 1731954455828Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@61762a9b: reopening flushed file at 1731954455927 (+99 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for fb8fa4293ea5e58c18dbc5b978355ca9 in 187ms, sequenceid=5, compaction requested=false at 1731954455979 (+52 ms)Writing region close event to WAL at 1731954456002 (+23 ms)Running coprocessor post-close hooks at 1731954456021 (+19 ms)Closed at 1731954456021 2024-11-18T18:27:36,021 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/69ae79c6f9eeed559fa88701f773706c/colfamily11/44975fa8c3d443418c8ec841193fee78, entries=703, sequenceid=5, filesize=38.3 K 2024-11-18T18:27:36,021 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731954260238.fb8fa4293ea5e58c18dbc5b978355ca9. 2024-11-18T18:27:36,021 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 97a938e093c14701871e85a1fbb6e07d, disabling compactions & flushes 2024-11-18T18:27:36,021 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731954260238.97a938e093c14701871e85a1fbb6e07d. 2024-11-18T18:27:36,021 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731954260238.97a938e093c14701871e85a1fbb6e07d. 2024-11-18T18:27:36,021 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731954260238.97a938e093c14701871e85a1fbb6e07d. after waiting 0 ms 2024-11-18T18:27:36,021 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731954260238.97a938e093c14701871e85a1fbb6e07d. 2024-11-18T18:27:36,022 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 97a938e093c14701871e85a1fbb6e07d 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-18T18:27:36,022 DEBUG [RS:2;961a81da9827:42895 {}] regionserver.HRegionServer(1351): Waiting on 43138a6a12c2efea7093064622bb3cd8, 97a938e093c14701871e85a1fbb6e07d, aaf2435418703b2a31c1190af1560ba8 2024-11-18T18:27:36,023 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 69ae79c6f9eeed559fa88701f773706c in 134ms, sequenceid=5, compaction requested=false 2024-11-18T18:27:36,049 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/69ae79c6f9eeed559fa88701f773706c/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-18T18:27:36,050 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731954260238.69ae79c6f9eeed559fa88701f773706c. 2024-11-18T18:27:36,051 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 69ae79c6f9eeed559fa88701f773706c: Waiting for close lock at 1731954455889Running coprocessor pre-close hooks at 1731954455889Disabling compacts and flushes for region at 1731954455889Disabling writes for close at 1731954455889Obtaining lock to block concurrent updates at 1731954455889Preparing flush snapshotting stores in 69ae79c6f9eeed559fa88701f773706c at 1731954455889Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731954260238.69ae79c6f9eeed559fa88701f773706c., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731954455890 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731954260238.69ae79c6f9eeed559fa88701f773706c. at 1731954455890Flushing 69ae79c6f9eeed559fa88701f773706c/colfamily11: creating writer at 1731954455890Flushing 69ae79c6f9eeed559fa88701f773706c/colfamily11: appending metadata at 1731954455915 (+25 ms)Flushing 69ae79c6f9eeed559fa88701f773706c/colfamily11: closing flushed file at 1731954455915Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@56d4ddad: reopening flushed file at 1731954456008 (+93 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 69ae79c6f9eeed559fa88701f773706c in 134ms, sequenceid=5, compaction requested=false at 1731954456023 (+15 ms)Writing region close event to WAL at 1731954456037 (+14 ms)Running coprocessor post-close hooks at 1731954456050 (+13 ms)Closed at 1731954456050 2024-11-18T18:27:36,051 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731954260238.69ae79c6f9eeed559fa88701f773706c. 2024-11-18T18:27:36,051 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing bcf8695060a326b7d24615ba6cb932b1, disabling compactions & flushes 2024-11-18T18:27:36,051 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731954260238.bcf8695060a326b7d24615ba6cb932b1. 2024-11-18T18:27:36,051 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731954260238.bcf8695060a326b7d24615ba6cb932b1. 2024-11-18T18:27:36,051 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731954260238.bcf8695060a326b7d24615ba6cb932b1. after waiting 0 ms 2024-11-18T18:27:36,051 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731954260238.bcf8695060a326b7d24615ba6cb932b1. 2024-11-18T18:27:36,068 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/97a938e093c14701871e85a1fbb6e07d/.tmp/colfamily11/c5af1e38a09a462c94a066886fef7a16 is 52, key is jjj/colfamily11:colfamily11/1731954264513/Put/seqid=0 2024-11-18T18:27:36,083 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bcf8695060a326b7d24615ba6cb932b1/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-18T18:27:36,084 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731954260238.bcf8695060a326b7d24615ba6cb932b1. 2024-11-18T18:27:36,084 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for bcf8695060a326b7d24615ba6cb932b1: Waiting for close lock at 1731954456051Running coprocessor pre-close hooks at 1731954456051Disabling compacts and flushes for region at 1731954456051Disabling writes for close at 1731954456051Writing region close event to WAL at 1731954456073 (+22 ms)Running coprocessor post-close hooks at 1731954456083 (+10 ms)Closed at 1731954456083 2024-11-18T18:27:36,084 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731954260238.bcf8695060a326b7d24615ba6cb932b1. 2024-11-18T18:27:36,084 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f39571a93e25091090ea0831d54936b5, disabling compactions & flushes 2024-11-18T18:27:36,084 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731954260238.f39571a93e25091090ea0831d54936b5. 2024-11-18T18:27:36,084 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731954260238.f39571a93e25091090ea0831d54936b5. 2024-11-18T18:27:36,084 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731954260238.f39571a93e25091090ea0831d54936b5. after waiting 0 ms 2024-11-18T18:27:36,084 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731954260238.f39571a93e25091090ea0831d54936b5. 2024-11-18T18:27:36,084 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing f39571a93e25091090ea0831d54936b5 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-18T18:27:36,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741902_1078 (size=39263) 2024-11-18T18:27:36,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741902_1078 (size=39263) 2024-11-18T18:27:36,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741902_1078 (size=39263) 2024-11-18T18:27:36,105 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/97a938e093c14701871e85a1fbb6e07d/.tmp/colfamily11/c5af1e38a09a462c94a066886fef7a16 2024-11-18T18:27:36,117 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/97a938e093c14701871e85a1fbb6e07d/.tmp/colfamily11/c5af1e38a09a462c94a066886fef7a16 as hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/97a938e093c14701871e85a1fbb6e07d/colfamily11/c5af1e38a09a462c94a066886fef7a16 2024-11-18T18:27:36,129 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/97a938e093c14701871e85a1fbb6e07d/colfamily11/c5af1e38a09a462c94a066886fef7a16, entries=703, sequenceid=5, filesize=38.3 K 2024-11-18T18:27:36,130 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 97a938e093c14701871e85a1fbb6e07d in 109ms, sequenceid=5, compaction requested=false 2024-11-18T18:27:36,131 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f39571a93e25091090ea0831d54936b5/.tmp/colfamily11/2d1c097eff244575834b714bb16c0cd5 is 52, key is iii/colfamily11:colfamily11/1731954264439/Put/seqid=0 2024-11-18T18:27:36,138 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/97a938e093c14701871e85a1fbb6e07d/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-18T18:27:36,139 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731954260238.97a938e093c14701871e85a1fbb6e07d. 2024-11-18T18:27:36,139 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 97a938e093c14701871e85a1fbb6e07d: Waiting for close lock at 1731954456021Running coprocessor pre-close hooks at 1731954456021Disabling compacts and flushes for region at 1731954456021Disabling writes for close at 1731954456021Obtaining lock to block concurrent updates at 1731954456022 (+1 ms)Preparing flush snapshotting stores in 97a938e093c14701871e85a1fbb6e07d at 1731954456022Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731954260238.97a938e093c14701871e85a1fbb6e07d., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731954456022Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731954260238.97a938e093c14701871e85a1fbb6e07d. at 1731954456023 (+1 ms)Flushing 97a938e093c14701871e85a1fbb6e07d/colfamily11: creating writer at 1731954456023Flushing 97a938e093c14701871e85a1fbb6e07d/colfamily11: appending metadata at 1731954456067 (+44 ms)Flushing 97a938e093c14701871e85a1fbb6e07d/colfamily11: closing flushed file at 1731954456068 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@fbc2d84: reopening flushed file at 1731954456113 (+45 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 97a938e093c14701871e85a1fbb6e07d in 109ms, sequenceid=5, compaction requested=false at 1731954456130 (+17 ms)Writing region close event to WAL at 1731954456133 (+3 ms)Running coprocessor post-close hooks at 1731954456139 (+6 ms)Closed at 1731954456139 2024-11-18T18:27:36,139 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731954260238.97a938e093c14701871e85a1fbb6e07d. 2024-11-18T18:27:36,139 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 43138a6a12c2efea7093064622bb3cd8, disabling compactions & flushes 2024-11-18T18:27:36,139 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731954260238.43138a6a12c2efea7093064622bb3cd8. 2024-11-18T18:27:36,139 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731954260238.43138a6a12c2efea7093064622bb3cd8. 2024-11-18T18:27:36,139 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731954260238.43138a6a12c2efea7093064622bb3cd8. after waiting 0 ms 2024-11-18T18:27:36,139 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731954260238.43138a6a12c2efea7093064622bb3cd8. 2024-11-18T18:27:36,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741903_1079 (size=39263) 2024-11-18T18:27:36,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741903_1079 (size=39263) 2024-11-18T18:27:36,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741903_1079 (size=39263) 2024-11-18T18:27:36,148 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/43138a6a12c2efea7093064622bb3cd8/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-18T18:27:36,149 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f39571a93e25091090ea0831d54936b5/.tmp/colfamily11/2d1c097eff244575834b714bb16c0cd5 2024-11-18T18:27:36,149 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731954260238.43138a6a12c2efea7093064622bb3cd8. 2024-11-18T18:27:36,149 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 43138a6a12c2efea7093064622bb3cd8: Waiting for close lock at 1731954456139Running coprocessor pre-close hooks at 1731954456139Disabling compacts and flushes for region at 1731954456139Disabling writes for close at 1731954456139Writing region close event to WAL at 1731954456144 (+5 ms)Running coprocessor post-close hooks at 1731954456149 (+5 ms)Closed at 1731954456149 2024-11-18T18:27:36,149 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731954260238.43138a6a12c2efea7093064622bb3cd8. 2024-11-18T18:27:36,150 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing aaf2435418703b2a31c1190af1560ba8, disabling compactions & flushes 2024-11-18T18:27:36,150 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731954260238.aaf2435418703b2a31c1190af1560ba8. 2024-11-18T18:27:36,150 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731954260238.aaf2435418703b2a31c1190af1560ba8. 2024-11-18T18:27:36,150 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731954260238.aaf2435418703b2a31c1190af1560ba8. after waiting 0 ms 2024-11-18T18:27:36,150 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731954260238.aaf2435418703b2a31c1190af1560ba8. 2024-11-18T18:27:36,150 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing aaf2435418703b2a31c1190af1560ba8 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-18T18:27:36,158 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f39571a93e25091090ea0831d54936b5/.tmp/colfamily11/2d1c097eff244575834b714bb16c0cd5 as hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f39571a93e25091090ea0831d54936b5/colfamily11/2d1c097eff244575834b714bb16c0cd5 2024-11-18T18:27:36,166 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f39571a93e25091090ea0831d54936b5/colfamily11/2d1c097eff244575834b714bb16c0cd5, entries=703, sequenceid=5, filesize=38.3 K 2024-11-18T18:27:36,167 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for f39571a93e25091090ea0831d54936b5 in 83ms, sequenceid=5, compaction requested=false 2024-11-18T18:27:36,178 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aaf2435418703b2a31c1190af1560ba8/.tmp/colfamily11/fcb4030c1174483782d20fe19ce94d20 is 52, key is ttt/colfamily11:colfamily11/1731954264715/Put/seqid=0 2024-11-18T18:27:36,178 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f39571a93e25091090ea0831d54936b5/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-18T18:27:36,180 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731954260238.f39571a93e25091090ea0831d54936b5. 2024-11-18T18:27:36,180 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f39571a93e25091090ea0831d54936b5: Waiting for close lock at 1731954456084Running coprocessor pre-close hooks at 1731954456084Disabling compacts and flushes for region at 1731954456084Disabling writes for close at 1731954456084Obtaining lock to block concurrent updates at 1731954456084Preparing flush snapshotting stores in f39571a93e25091090ea0831d54936b5 at 1731954456084Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731954260238.f39571a93e25091090ea0831d54936b5., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731954456085 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731954260238.f39571a93e25091090ea0831d54936b5. at 1731954456085Flushing f39571a93e25091090ea0831d54936b5/colfamily11: creating writer at 1731954456085Flushing f39571a93e25091090ea0831d54936b5/colfamily11: appending metadata at 1731954456131 (+46 ms)Flushing f39571a93e25091090ea0831d54936b5/colfamily11: closing flushed file at 1731954456131Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77b97a1d: reopening flushed file at 1731954456157 (+26 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for f39571a93e25091090ea0831d54936b5 in 83ms, sequenceid=5, compaction requested=false at 1731954456167 (+10 ms)Writing region close event to WAL at 1731954456173 (+6 ms)Running coprocessor post-close hooks at 1731954456180 (+7 ms)Closed at 1731954456180 2024-11-18T18:27:36,180 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731954260238.f39571a93e25091090ea0831d54936b5. 2024-11-18T18:27:36,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741904_1080 (size=39263) 2024-11-18T18:27:36,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741904_1080 (size=39263) 2024-11-18T18:27:36,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741904_1080 (size=39263) 2024-11-18T18:27:36,201 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aaf2435418703b2a31c1190af1560ba8/.tmp/colfamily11/fcb4030c1174483782d20fe19ce94d20 2024-11-18T18:27:36,208 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aaf2435418703b2a31c1190af1560ba8/.tmp/colfamily11/fcb4030c1174483782d20fe19ce94d20 as hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aaf2435418703b2a31c1190af1560ba8/colfamily11/fcb4030c1174483782d20fe19ce94d20 2024-11-18T18:27:36,214 INFO [RS:0;961a81da9827:38709 {}] regionserver.HRegionServer(976): stopping server 961a81da9827,38709,1731954257723; all regions closed. 2024-11-18T18:27:36,214 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aaf2435418703b2a31c1190af1560ba8/colfamily11/fcb4030c1174483782d20fe19ce94d20, entries=703, sequenceid=5, filesize=38.3 K 2024-11-18T18:27:36,216 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for aaf2435418703b2a31c1190af1560ba8 in 66ms, sequenceid=5, compaction requested=false 2024-11-18T18:27:36,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741835_1011 (size=277788) 2024-11-18T18:27:36,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741835_1011 (size=277788) 2024-11-18T18:27:36,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741835_1011 (size=277788) 2024-11-18T18:27:36,221 DEBUG [RS:0;961a81da9827:38709 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/oldWALs 2024-11-18T18:27:36,221 INFO [RS:0;961a81da9827:38709 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 961a81da9827%2C38709%2C1731954257723:(num 1731954259308) 2024-11-18T18:27:36,221 DEBUG [RS:0;961a81da9827:38709 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T18:27:36,221 INFO [RS:0;961a81da9827:38709 {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T18:27:36,221 INFO [RS:0;961a81da9827:38709 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T18:27:36,221 INFO [RS:0;961a81da9827:38709 {}] hbase.ChoreService(370): Chore service for: regionserver/961a81da9827:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-18T18:27:36,221 INFO [RS:0;961a81da9827:38709 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-18T18:27:36,222 INFO [RS:0;961a81da9827:38709 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-18T18:27:36,222 INFO [RS:0;961a81da9827:38709 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-18T18:27:36,222 INFO [RS:0;961a81da9827:38709 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T18:27:36,222 INFO [RS:0;961a81da9827:38709 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38709 2024-11-18T18:27:36,222 INFO [regionserver/961a81da9827:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T18:27:36,222 DEBUG [RS:2;961a81da9827:42895 {}] regionserver.HRegionServer(1351): Waiting on aaf2435418703b2a31c1190af1560ba8 2024-11-18T18:27:36,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38709-0x10107179d9f0001, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/961a81da9827,38709,1731954257723 2024-11-18T18:27:36,224 INFO [RS:0;961a81da9827:38709 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T18:27:36,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33089-0x10107179d9f0000, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T18:27:36,226 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [961a81da9827,38709,1731954257723] 2024-11-18T18:27:36,227 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/961a81da9827,38709,1731954257723 already deleted, retry=false 2024-11-18T18:27:36,228 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 961a81da9827,38709,1731954257723 expired; onlineServers=1 2024-11-18T18:27:36,235 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aaf2435418703b2a31c1190af1560ba8/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-18T18:27:36,236 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731954260238.aaf2435418703b2a31c1190af1560ba8. 2024-11-18T18:27:36,236 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for aaf2435418703b2a31c1190af1560ba8: Waiting for close lock at 1731954456150Running coprocessor pre-close hooks at 1731954456150Disabling compacts and flushes for region at 1731954456150Disabling writes for close at 1731954456150Obtaining lock to block concurrent updates at 1731954456150Preparing flush snapshotting stores in aaf2435418703b2a31c1190af1560ba8 at 1731954456150Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731954260238.aaf2435418703b2a31c1190af1560ba8., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731954456150Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731954260238.aaf2435418703b2a31c1190af1560ba8. at 1731954456152 (+2 ms)Flushing aaf2435418703b2a31c1190af1560ba8/colfamily11: creating writer at 1731954456152Flushing aaf2435418703b2a31c1190af1560ba8/colfamily11: appending metadata at 1731954456177 (+25 ms)Flushing aaf2435418703b2a31c1190af1560ba8/colfamily11: closing flushed file at 1731954456177Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1af13617: reopening flushed file at 1731954456207 (+30 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for aaf2435418703b2a31c1190af1560ba8 in 66ms, sequenceid=5, compaction requested=false at 1731954456216 (+9 ms)Writing region close event to WAL at 1731954456230 (+14 ms)Running coprocessor post-close hooks at 1731954456236 (+6 ms)Closed at 1731954456236 2024-11-18T18:27:36,236 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731954260238.aaf2435418703b2a31c1190af1560ba8. 2024-11-18T18:27:36,326 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38709-0x10107179d9f0001, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T18:27:36,326 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38709-0x10107179d9f0001, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T18:27:36,326 INFO [RS:0;961a81da9827:38709 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T18:27:36,326 INFO [RS:0;961a81da9827:38709 {}] regionserver.HRegionServer(1031): Exiting; stopping=961a81da9827,38709,1731954257723; zookeeper connection closed. 2024-11-18T18:27:36,327 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@403aa47e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@403aa47e 2024-11-18T18:27:36,423 INFO [RS:2;961a81da9827:42895 {}] regionserver.HRegionServer(976): stopping server 961a81da9827,42895,1731954257867; all regions closed. 2024-11-18T18:27:36,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741865_1041 (size=15752) 2024-11-18T18:27:36,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741865_1041 (size=15752) 2024-11-18T18:27:36,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741865_1041 (size=15752) 2024-11-18T18:27:36,429 DEBUG [RS:2;961a81da9827:42895 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/oldWALs 2024-11-18T18:27:36,429 INFO [RS:2;961a81da9827:42895 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 961a81da9827%2C42895%2C1731954257867.meta:.meta(num 1731954445967) 2024-11-18T18:27:36,432 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/WALs/961a81da9827,42895,1731954257867/961a81da9827%2C42895%2C1731954257867.1731954259295 not finished, retry = 0 2024-11-18T18:27:36,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741834_1010 (size=354137) 2024-11-18T18:27:36,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741834_1010 (size=354137) 2024-11-18T18:27:36,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741834_1010 (size=354137) 2024-11-18T18:27:36,534 DEBUG [RS:2;961a81da9827:42895 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/oldWALs 2024-11-18T18:27:36,534 INFO [RS:2;961a81da9827:42895 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 961a81da9827%2C42895%2C1731954257867:(num 1731954259295) 2024-11-18T18:27:36,534 DEBUG [RS:2;961a81da9827:42895 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T18:27:36,534 INFO [RS:2;961a81da9827:42895 {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T18:27:36,535 INFO [RS:2;961a81da9827:42895 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T18:27:36,535 INFO [RS:2;961a81da9827:42895 {}] hbase.ChoreService(370): Chore service for: regionserver/961a81da9827:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-18T18:27:36,535 INFO [RS:2;961a81da9827:42895 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T18:27:36,535 INFO [regionserver/961a81da9827:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T18:27:36,535 INFO [RS:2;961a81da9827:42895 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42895 2024-11-18T18:27:36,537 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42895-0x10107179d9f0003, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/961a81da9827,42895,1731954257867 2024-11-18T18:27:36,537 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33089-0x10107179d9f0000, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T18:27:36,537 INFO [RS:2;961a81da9827:42895 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T18:27:36,539 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [961a81da9827,42895,1731954257867] 2024-11-18T18:27:36,540 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/961a81da9827,42895,1731954257867 already deleted, retry=false 2024-11-18T18:27:36,540 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 961a81da9827,42895,1731954257867 expired; onlineServers=0 2024-11-18T18:27:36,541 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '961a81da9827,33089,1731954257001' ***** 2024-11-18T18:27:36,541 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-18T18:27:36,541 INFO [M:0;961a81da9827:33089 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T18:27:36,541 INFO [M:0;961a81da9827:33089 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T18:27:36,541 DEBUG [M:0;961a81da9827:33089 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-18T18:27:36,541 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-18T18:27:36,541 DEBUG [M:0;961a81da9827:33089 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-18T18:27:36,541 DEBUG [master/961a81da9827:0:becomeActiveMaster-HFileCleaner.small.0-1731954258897 {}] cleaner.HFileCleaner(306): Exit Thread[master/961a81da9827:0:becomeActiveMaster-HFileCleaner.small.0-1731954258897,5,FailOnTimeoutGroup] 2024-11-18T18:27:36,541 DEBUG [master/961a81da9827:0:becomeActiveMaster-HFileCleaner.large.0-1731954258896 {}] cleaner.HFileCleaner(306): Exit Thread[master/961a81da9827:0:becomeActiveMaster-HFileCleaner.large.0-1731954258896,5,FailOnTimeoutGroup] 2024-11-18T18:27:36,541 INFO [M:0;961a81da9827:33089 {}] hbase.ChoreService(370): Chore service for: master/961a81da9827:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-18T18:27:36,541 INFO [M:0;961a81da9827:33089 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T18:27:36,541 DEBUG [M:0;961a81da9827:33089 {}] master.HMaster(1795): Stopping service threads 2024-11-18T18:27:36,541 INFO [M:0;961a81da9827:33089 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-18T18:27:36,542 INFO [M:0;961a81da9827:33089 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T18:27:36,542 INFO [M:0;961a81da9827:33089 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-18T18:27:36,542 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-18T18:27:36,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33089-0x10107179d9f0000, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-18T18:27:36,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33089-0x10107179d9f0000, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:27:36,543 DEBUG [M:0;961a81da9827:33089 {}] zookeeper.ZKUtil(347): master:33089-0x10107179d9f0000, quorum=127.0.0.1:59292, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-18T18:27:36,544 WARN [M:0;961a81da9827:33089 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-18T18:27:36,544 INFO [M:0;961a81da9827:33089 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/.lastflushedseqids 2024-11-18T18:27:36,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741905_1081 (size=2837) 2024-11-18T18:27:36,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741905_1081 (size=2837) 2024-11-18T18:27:36,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741905_1081 (size=2837) 2024-11-18T18:27:36,558 INFO [M:0;961a81da9827:33089 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-18T18:27:36,558 INFO [M:0;961a81da9827:33089 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-18T18:27:36,559 DEBUG [M:0;961a81da9827:33089 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-18T18:27:36,559 INFO [M:0;961a81da9827:33089 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T18:27:36,559 DEBUG [M:0;961a81da9827:33089 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T18:27:36,559 DEBUG [M:0;961a81da9827:33089 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-18T18:27:36,559 DEBUG [M:0;961a81da9827:33089 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T18:27:36,559 INFO [M:0;961a81da9827:33089 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=342.63 KB heapSize=402.82 KB 2024-11-18T18:27:36,583 DEBUG [M:0;961a81da9827:33089 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1e18cb03c98c46ffa2dcfb9d83ac5181 is 82, key is hbase:meta,,1/info:regioninfo/1731954446461/Put/seqid=0 2024-11-18T18:27:36,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741906_1082 (size=6071) 2024-11-18T18:27:36,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741906_1082 (size=6071) 2024-11-18T18:27:36,593 INFO [M:0;961a81da9827:33089 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=632 (bloomFilter=true), to=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1e18cb03c98c46ffa2dcfb9d83ac5181 2024-11-18T18:27:36,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741906_1082 (size=6071) 2024-11-18T18:27:36,623 DEBUG [M:0;961a81da9827:33089 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/781ff659947f467fb06be27b71a9c94a is 4106, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731954262174/Put/seqid=0 2024-11-18T18:27:36,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741907_1083 (size=13506) 2024-11-18T18:27:36,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741907_1083 (size=13506) 2024-11-18T18:27:36,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741907_1083 (size=13506) 2024-11-18T18:27:36,631 INFO [M:0;961a81da9827:33089 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=341.40 KB at sequenceid=632 (bloomFilter=true), to=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/781ff659947f467fb06be27b71a9c94a 2024-11-18T18:27:36,639 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42895-0x10107179d9f0003, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T18:27:36,639 INFO [RS:2;961a81da9827:42895 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T18:27:36,639 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42895-0x10107179d9f0003, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T18:27:36,639 INFO [RS:2;961a81da9827:42895 {}] regionserver.HRegionServer(1031): Exiting; stopping=961a81da9827,42895,1731954257867; zookeeper connection closed. 2024-11-18T18:27:36,640 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@75f3afe {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@75f3afe 2024-11-18T18:27:36,640 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-18T18:27:36,652 DEBUG [M:0;961a81da9827:33089 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2e887a0fe68741c1a524f26e6d716aef is 69, key is 961a81da9827,38709,1731954257723/rs:state/1731954258987/Put/seqid=0 2024-11-18T18:27:36,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741908_1084 (size=5442) 2024-11-18T18:27:36,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741908_1084 (size=5442) 2024-11-18T18:27:36,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741908_1084 (size=5442) 2024-11-18T18:27:36,659 INFO [M:0;961a81da9827:33089 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=632 (bloomFilter=true), to=hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2e887a0fe68741c1a524f26e6d716aef 2024-11-18T18:27:36,664 INFO [M:0;961a81da9827:33089 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2e887a0fe68741c1a524f26e6d716aef 2024-11-18T18:27:36,664 DEBUG [M:0;961a81da9827:33089 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1e18cb03c98c46ffa2dcfb9d83ac5181 as hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1e18cb03c98c46ffa2dcfb9d83ac5181 2024-11-18T18:27:36,669 INFO [M:0;961a81da9827:33089 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1e18cb03c98c46ffa2dcfb9d83ac5181, entries=14, sequenceid=632, filesize=5.9 K 2024-11-18T18:27:36,670 DEBUG [M:0;961a81da9827:33089 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/781ff659947f467fb06be27b71a9c94a as hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/781ff659947f467fb06be27b71a9c94a 2024-11-18T18:27:36,674 INFO [M:0;961a81da9827:33089 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/781ff659947f467fb06be27b71a9c94a, entries=81, sequenceid=632, filesize=13.2 K 2024-11-18T18:27:36,675 DEBUG [M:0;961a81da9827:33089 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2e887a0fe68741c1a524f26e6d716aef as hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2e887a0fe68741c1a524f26e6d716aef 2024-11-18T18:27:36,679 INFO [M:0;961a81da9827:33089 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2e887a0fe68741c1a524f26e6d716aef 2024-11-18T18:27:36,679 INFO [M:0;961a81da9827:33089 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43935/user/jenkins/test-data/408d680f-ec0d-d4df-808c-96f793db9c93/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2e887a0fe68741c1a524f26e6d716aef, entries=3, sequenceid=632, filesize=5.3 K 2024-11-18T18:27:36,680 INFO [M:0;961a81da9827:33089 {}] regionserver.HRegion(3140): Finished flush of dataSize ~342.63 KB/350852, heapSize ~402.52 KB/412184, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 121ms, sequenceid=632, compaction requested=false 2024-11-18T18:27:36,681 INFO [M:0;961a81da9827:33089 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T18:27:36,681 DEBUG [M:0;961a81da9827:33089 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731954456558Disabling compacts and flushes for region at 1731954456558Disabling writes for close at 1731954456559 (+1 ms)Obtaining lock to block concurrent updates at 1731954456559Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731954456559Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=350852, getHeapSize=412424, getOffHeapSize=0, getCellsCount=797 at 1731954456559Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731954456560 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731954456560Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731954456583 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731954456583Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731954456599 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731954456622 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731954456622Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731954456636 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731954456651 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731954456651Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1921c2cb: reopening flushed file at 1731954456664 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a1fae7d: reopening flushed file at 1731954456669 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5706228b: reopening flushed file at 1731954456674 (+5 ms)Finished flush of dataSize ~342.63 KB/350852, heapSize ~402.52 KB/412184, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 121ms, sequenceid=632, compaction requested=false at 1731954456680 (+6 ms)Writing region close event to WAL at 1731954456681 (+1 ms)Closed at 1731954456681 2024-11-18T18:27:36,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34047 is added to blk_1073741830_1006 (size=393011) 2024-11-18T18:27:36,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41713 is added to blk_1073741830_1006 (size=393011) 2024-11-18T18:27:36,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741830_1006 (size=393011) 2024-11-18T18:27:36,685 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T18:27:36,685 INFO [M:0;961a81da9827:33089 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-18T18:27:36,685 INFO [M:0;961a81da9827:33089 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33089 2024-11-18T18:27:36,685 INFO [M:0;961a81da9827:33089 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T18:27:36,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33089-0x10107179d9f0000, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T18:27:36,787 INFO [M:0;961a81da9827:33089 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T18:27:36,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33089-0x10107179d9f0000, quorum=127.0.0.1:59292, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T18:27:36,792 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6478847d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-18T18:27:36,794 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@15e91890{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-18T18:27:36,794 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-18T18:27:36,794 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7f9dc0ae{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-18T18:27:36,794 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@56f216fb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67b3225c-0ec5-86bb-7544-cb0f1a3b69d6/hadoop.log.dir/,STOPPED} 2024-11-18T18:27:36,796 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-18T18:27:36,796 WARN [BP-868939734-172.17.0.2-1731954253414 heartbeating to localhost/127.0.0.1:43935 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-18T18:27:36,796 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-18T18:27:36,796 WARN [BP-868939734-172.17.0.2-1731954253414 heartbeating to localhost/127.0.0.1:43935 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-868939734-172.17.0.2-1731954253414 (Datanode Uuid 7f99a8ef-4a0d-4d55-b1dd-487bc0910b13) service to localhost/127.0.0.1:43935 2024-11-18T18:27:36,798 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67b3225c-0ec5-86bb-7544-cb0f1a3b69d6/cluster_cb6102b6-aad9-a0f9-bd1c-aa9450167237/data/data5/current/BP-868939734-172.17.0.2-1731954253414 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-18T18:27:36,798 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67b3225c-0ec5-86bb-7544-cb0f1a3b69d6/cluster_cb6102b6-aad9-a0f9-bd1c-aa9450167237/data/data6/current/BP-868939734-172.17.0.2-1731954253414 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-18T18:27:36,799 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-18T18:27:36,801 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@666ae17c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-18T18:27:36,801 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@270be46{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-18T18:27:36,801 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-18T18:27:36,801 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7186ce9f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-18T18:27:36,801 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@57794ee2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67b3225c-0ec5-86bb-7544-cb0f1a3b69d6/hadoop.log.dir/,STOPPED} 2024-11-18T18:27:36,805 WARN [BP-868939734-172.17.0.2-1731954253414 heartbeating to localhost/127.0.0.1:43935 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-18T18:27:36,805 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-18T18:27:36,805 WARN [BP-868939734-172.17.0.2-1731954253414 heartbeating to localhost/127.0.0.1:43935 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-868939734-172.17.0.2-1731954253414 (Datanode Uuid 76d78cf9-913a-4743-9820-1225728dbf25) service to localhost/127.0.0.1:43935 2024-11-18T18:27:36,805 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-18T18:27:36,806 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67b3225c-0ec5-86bb-7544-cb0f1a3b69d6/cluster_cb6102b6-aad9-a0f9-bd1c-aa9450167237/data/data3/current/BP-868939734-172.17.0.2-1731954253414 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-18T18:27:36,807 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67b3225c-0ec5-86bb-7544-cb0f1a3b69d6/cluster_cb6102b6-aad9-a0f9-bd1c-aa9450167237/data/data4/current/BP-868939734-172.17.0.2-1731954253414 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-18T18:27:36,807 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-18T18:27:36,809 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6f328155{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-18T18:27:36,809 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6fd8ef10{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-18T18:27:36,809 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-18T18:27:36,809 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93e0721{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-18T18:27:36,809 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@463823fd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67b3225c-0ec5-86bb-7544-cb0f1a3b69d6/hadoop.log.dir/,STOPPED} 2024-11-18T18:27:36,813 WARN [BP-868939734-172.17.0.2-1731954253414 heartbeating to localhost/127.0.0.1:43935 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-18T18:27:36,813 WARN [BP-868939734-172.17.0.2-1731954253414 heartbeating to localhost/127.0.0.1:43935 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-868939734-172.17.0.2-1731954253414 (Datanode Uuid 5a9c9d5e-ee42-47fc-ab8f-e8ed647e5b1c) service to localhost/127.0.0.1:43935 2024-11-18T18:27:36,813 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-18T18:27:36,813 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-18T18:27:36,814 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67b3225c-0ec5-86bb-7544-cb0f1a3b69d6/cluster_cb6102b6-aad9-a0f9-bd1c-aa9450167237/data/data1/current/BP-868939734-172.17.0.2-1731954253414 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-18T18:27:36,814 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67b3225c-0ec5-86bb-7544-cb0f1a3b69d6/cluster_cb6102b6-aad9-a0f9-bd1c-aa9450167237/data/data2/current/BP-868939734-172.17.0.2-1731954253414 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-18T18:27:36,815 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-18T18:27:36,827 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5183f126{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-18T18:27:36,828 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4350931a{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-18T18:27:36,828 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-18T18:27:36,828 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4974766f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-18T18:27:36,828 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2c447bb0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/67b3225c-0ec5-86bb-7544-cb0f1a3b69d6/hadoop.log.dir/,STOPPED} 2024-11-18T18:27:36,842 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-18T18:27:36,921 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-18T18:27:36,932 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.procedure.TestSCPWithMeta#testRecoveryAndDoubleExecutionOnRsWithMeta Thread=88 (was 11) Potentially hanging thread: IPC Client (470560621) connection to localhost/127.0.0.1:43935 from jenkins.hfs.0 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-6-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: nioEventLoopGroup-4-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.0@localhost:43935 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-5-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/961a81da9827:0.Chore.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/961a81da9827:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: nioEventLoopGroup-3-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-5-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.metrics2.impl.JmxCacheBuster$JmxCacheBusterRunnable.run(JmxCacheBuster.java:101) java.base@17.0.11/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) java.base@17.0.11/java.util.concurrent.FutureTask.run(FutureTask.java:264) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-7-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_META-regionserver/961a81da9827:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161) Potentially hanging thread: nioEventLoopGroup-6-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (470560621) connection to localhost/127.0.0.1:43935 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43935 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (470560621) connection to localhost/127.0.0.1:43935 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-2-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/961a81da9827:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@3210dba6 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253) app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SSL Certificates Store Monitor java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: nioEventLoopGroup-4-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/961a81da9827:0.logRoller java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:180) Potentially hanging thread: ForkJoinPool-2-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: region-location-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/961a81da9827:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.2@localhost:43935 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-7-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:43935 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-5-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-7-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: BootstrapNodeManager java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/961a81da9827:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Abort regionserver monitor java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/961a81da9827:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (470560621) connection to localhost/127.0.0.1:43935 from jenkins.hfs.2 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: RS_OPEN_REGION-regionserver/961a81da9827:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-6-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43935 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43935 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43935 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=458 (was 285) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=897 (was 607) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1015 (was 3808)