2024-11-13 16:08:00,710 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-13 16:08:00,748 main DEBUG Took 0.034855 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-13 16:08:00,748 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-13 16:08:00,749 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-13 16:08:00,751 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-13 16:08:00,753 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 16:08:00,766 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-13 16:08:00,785 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 16:08:00,787 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 16:08:00,788 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 16:08:00,789 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 16:08:00,790 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 16:08:00,790 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 16:08:00,791 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 16:08:00,792 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 16:08:00,793 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 16:08:00,793 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 16:08:00,794 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 16:08:00,795 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 16:08:00,796 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 16:08:00,796 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 16:08:00,797 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 16:08:00,797 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 16:08:00,798 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 16:08:00,799 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 16:08:00,799 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 16:08:00,800 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 16:08:00,800 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 16:08:00,801 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 16:08:00,802 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 16:08:00,802 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 16:08:00,803 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 16:08:00,803 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-13 16:08:00,805 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 16:08:00,807 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-13 16:08:00,810 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-13 16:08:00,811 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-13 16:08:00,813 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-13 16:08:00,813 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-13 16:08:00,825 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-13 16:08:00,829 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-13 16:08:00,831 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-13 16:08:00,832 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-13 16:08:00,832 main DEBUG createAppenders(={Console}) 2024-11-13 16:08:00,833 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-13 16:08:00,834 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-13 16:08:00,834 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-13 16:08:00,835 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-13 16:08:00,835 main DEBUG OutputStream closed 2024-11-13 16:08:00,835 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-13 16:08:00,836 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-13 16:08:00,836 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-13 16:08:00,965 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-13 16:08:00,969 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-13 16:08:00,970 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-13 16:08:00,972 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-13 16:08:00,972 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-13 16:08:00,973 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-13 16:08:00,974 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-13 16:08:00,974 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-13 16:08:00,975 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-13 16:08:00,976 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-13 16:08:00,976 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-13 16:08:00,977 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-13 16:08:00,977 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-13 16:08:00,978 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-13 16:08:00,978 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-13 16:08:00,979 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-13 16:08:00,979 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-13 16:08:00,981 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-13 16:08:00,984 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13 16:08:00,985 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-13 16:08:00,986 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-13 16:08:00,988 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-13T16:08:01,009 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.procedure.TestSCPWithMeta timeout: 13 mins 2024-11-13 16:08:01,013 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-13 16:08:01,014 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13T16:08:01,085 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.procedure.TestSCPWithMeta#testRecoveryAndDoubleExecutionOnRsWithMeta Thread=11, OpenFileDescriptor=285, MaxFileDescriptor=1048576, SystemLoadAverage=938, ProcessCount=11, AvailableMemoryMB=3482 2024-11-13T16:08:01,541 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3772cfda-71de-26d3-3229-45fcb377a4d9 2024-11-13T16:08:01,542 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-13T16:08:01,582 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3772cfda-71de-26d3-3229-45fcb377a4d9/cluster_1b15728d-283e-8daa-5d9c-302860a3b764, deleteOnExit=true 2024-11-13T16:08:01,583 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-13T16:08:01,587 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3772cfda-71de-26d3-3229-45fcb377a4d9/test.cache.data in system properties and HBase conf 2024-11-13T16:08:01,588 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3772cfda-71de-26d3-3229-45fcb377a4d9/hadoop.tmp.dir in system properties and HBase conf 2024-11-13T16:08:01,589 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3772cfda-71de-26d3-3229-45fcb377a4d9/hadoop.log.dir in system properties and HBase conf 2024-11-13T16:08:01,590 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3772cfda-71de-26d3-3229-45fcb377a4d9/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-13T16:08:01,591 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3772cfda-71de-26d3-3229-45fcb377a4d9/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-13T16:08:01,591 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-13T16:08:01,701 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-13T16:08:01,816 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T16:08:01,822 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3772cfda-71de-26d3-3229-45fcb377a4d9/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-13T16:08:01,823 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3772cfda-71de-26d3-3229-45fcb377a4d9/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-13T16:08:01,823 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3772cfda-71de-26d3-3229-45fcb377a4d9/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-13T16:08:01,824 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3772cfda-71de-26d3-3229-45fcb377a4d9/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T16:08:01,825 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3772cfda-71de-26d3-3229-45fcb377a4d9/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-13T16:08:01,826 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3772cfda-71de-26d3-3229-45fcb377a4d9/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-13T16:08:01,826 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3772cfda-71de-26d3-3229-45fcb377a4d9/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T16:08:01,827 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3772cfda-71de-26d3-3229-45fcb377a4d9/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T16:08:01,828 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3772cfda-71de-26d3-3229-45fcb377a4d9/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-13T16:08:01,828 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3772cfda-71de-26d3-3229-45fcb377a4d9/nfs.dump.dir in system properties and HBase conf 2024-11-13T16:08:01,829 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3772cfda-71de-26d3-3229-45fcb377a4d9/java.io.tmpdir in system properties and HBase conf 2024-11-13T16:08:01,830 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3772cfda-71de-26d3-3229-45fcb377a4d9/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T16:08:01,830 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3772cfda-71de-26d3-3229-45fcb377a4d9/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-13T16:08:01,831 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3772cfda-71de-26d3-3229-45fcb377a4d9/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-13T16:08:03,213 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-13T16:08:03,307 INFO [Time-limited test {}] log.Log(170): Logging initialized @3522ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-13T16:08:03,388 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T16:08:03,473 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T16:08:03,512 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T16:08:03,512 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T16:08:03,514 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T16:08:03,534 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T16:08:03,539 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@34f5021e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3772cfda-71de-26d3-3229-45fcb377a4d9/hadoop.log.dir/,AVAILABLE} 2024-11-13T16:08:03,540 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4164510b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T16:08:03,779 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6c8e2b9b{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3772cfda-71de-26d3-3229-45fcb377a4d9/java.io.tmpdir/jetty-localhost-40003-hadoop-hdfs-3_4_1-tests_jar-_-any-14134257019877760605/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T16:08:03,791 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@cd7e66f{HTTP/1.1, (http/1.1)}{localhost:40003} 2024-11-13T16:08:03,791 INFO [Time-limited test {}] server.Server(415): Started @4007ms 2024-11-13T16:08:04,460 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T16:08:04,472 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T16:08:04,478 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T16:08:04,478 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T16:08:04,479 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T16:08:04,480 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@31fa71ec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3772cfda-71de-26d3-3229-45fcb377a4d9/hadoop.log.dir/,AVAILABLE} 2024-11-13T16:08:04,481 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7a83e474{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T16:08:04,594 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@75313660{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3772cfda-71de-26d3-3229-45fcb377a4d9/java.io.tmpdir/jetty-localhost-42413-hadoop-hdfs-3_4_1-tests_jar-_-any-12489053763533806404/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T16:08:04,594 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@10d359de{HTTP/1.1, (http/1.1)}{localhost:42413} 2024-11-13T16:08:04,595 INFO [Time-limited test {}] server.Server(415): Started @4810ms 2024-11-13T16:08:04,657 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T16:08:04,862 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T16:08:04,873 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T16:08:04,879 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T16:08:04,879 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T16:08:04,880 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T16:08:04,882 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3a2ddbb3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3772cfda-71de-26d3-3229-45fcb377a4d9/hadoop.log.dir/,AVAILABLE} 2024-11-13T16:08:04,886 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@62535b13{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T16:08:05,028 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@60b2194c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3772cfda-71de-26d3-3229-45fcb377a4d9/java.io.tmpdir/jetty-localhost-45797-hadoop-hdfs-3_4_1-tests_jar-_-any-14526462585398522833/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T16:08:05,029 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7ac58d5{HTTP/1.1, (http/1.1)}{localhost:45797} 2024-11-13T16:08:05,029 INFO [Time-limited test {}] server.Server(415): Started @5245ms 2024-11-13T16:08:05,033 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T16:08:05,080 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T16:08:05,088 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T16:08:05,089 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T16:08:05,090 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T16:08:05,090 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T16:08:05,091 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@697bf951{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3772cfda-71de-26d3-3229-45fcb377a4d9/hadoop.log.dir/,AVAILABLE} 2024-11-13T16:08:05,092 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7605ef0c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T16:08:05,231 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@47300910{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3772cfda-71de-26d3-3229-45fcb377a4d9/java.io.tmpdir/jetty-localhost-35759-hadoop-hdfs-3_4_1-tests_jar-_-any-11696992448544212353/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T16:08:05,232 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@38cb4984{HTTP/1.1, (http/1.1)}{localhost:35759} 2024-11-13T16:08:05,233 INFO [Time-limited test {}] server.Server(415): Started @5448ms 2024-11-13T16:08:05,235 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T16:08:06,360 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3772cfda-71de-26d3-3229-45fcb377a4d9/cluster_1b15728d-283e-8daa-5d9c-302860a3b764/data/data5/current/BP-1848284120-172.17.0.3-1731514082621/current, will proceed with Du for space computation calculation, 2024-11-13T16:08:06,363 WARN [Thread-126 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3772cfda-71de-26d3-3229-45fcb377a4d9/cluster_1b15728d-283e-8daa-5d9c-302860a3b764/data/data1/current/BP-1848284120-172.17.0.3-1731514082621/current, will proceed with Du for space computation calculation, 2024-11-13T16:08:06,366 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3772cfda-71de-26d3-3229-45fcb377a4d9/cluster_1b15728d-283e-8daa-5d9c-302860a3b764/data/data3/current/BP-1848284120-172.17.0.3-1731514082621/current, will proceed with Du for space computation calculation, 2024-11-13T16:08:06,366 WARN [Thread-127 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3772cfda-71de-26d3-3229-45fcb377a4d9/cluster_1b15728d-283e-8daa-5d9c-302860a3b764/data/data6/current/BP-1848284120-172.17.0.3-1731514082621/current, will proceed with Du for space computation calculation, 2024-11-13T16:08:06,367 WARN [Thread-128 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3772cfda-71de-26d3-3229-45fcb377a4d9/cluster_1b15728d-283e-8daa-5d9c-302860a3b764/data/data4/current/BP-1848284120-172.17.0.3-1731514082621/current, will proceed with Du for space computation calculation, 2024-11-13T16:08:06,367 WARN [Thread-129 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3772cfda-71de-26d3-3229-45fcb377a4d9/cluster_1b15728d-283e-8daa-5d9c-302860a3b764/data/data2/current/BP-1848284120-172.17.0.3-1731514082621/current, will proceed with Du for space computation calculation, 2024-11-13T16:08:06,432 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T16:08:06,432 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T16:08:06,432 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T16:08:06,479 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb6ab7af76ac8a0ab with lease ID 0xe17620f2bfeeb2b7: Processing first storage report for DS-b848e0f5-c5b4-4830-9d9c-fc9502100de9 from datanode DatanodeRegistration(127.0.0.1:45743, datanodeUuid=3de4e64d-0c3f-4dd4-b7a5-e67576d85d5e, infoPort=46455, infoSecurePort=0, ipcPort=33353, storageInfo=lv=-57;cid=testClusterID;nsid=1240587590;c=1731514082621) 2024-11-13T16:08:06,480 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb6ab7af76ac8a0ab with lease ID 0xe17620f2bfeeb2b7: from storage DS-b848e0f5-c5b4-4830-9d9c-fc9502100de9 node DatanodeRegistration(127.0.0.1:45743, datanodeUuid=3de4e64d-0c3f-4dd4-b7a5-e67576d85d5e, infoPort=46455, infoSecurePort=0, ipcPort=33353, storageInfo=lv=-57;cid=testClusterID;nsid=1240587590;c=1731514082621), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-13T16:08:06,480 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x259e61375e0914da with lease ID 0xe17620f2bfeeb2b5: Processing first storage report for DS-af4262a8-b437-4e57-821f-0315c01636b9 from datanode DatanodeRegistration(127.0.0.1:32863, datanodeUuid=fc672ff6-9eb1-4fc8-a112-e265d00093c2, infoPort=33713, infoSecurePort=0, ipcPort=43595, storageInfo=lv=-57;cid=testClusterID;nsid=1240587590;c=1731514082621) 2024-11-13T16:08:06,480 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x259e61375e0914da with lease ID 0xe17620f2bfeeb2b5: from storage DS-af4262a8-b437-4e57-821f-0315c01636b9 node DatanodeRegistration(127.0.0.1:32863, datanodeUuid=fc672ff6-9eb1-4fc8-a112-e265d00093c2, infoPort=33713, infoSecurePort=0, ipcPort=43595, storageInfo=lv=-57;cid=testClusterID;nsid=1240587590;c=1731514082621), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T16:08:06,480 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb01705cc8c5692ae with lease ID 0xe17620f2bfeeb2b6: Processing first storage report for DS-83f68ee5-197b-4527-968f-c0dc23954597 from datanode DatanodeRegistration(127.0.0.1:33087, datanodeUuid=4c2e890c-e34f-48b7-b831-a71ec0b9834d, infoPort=38919, infoSecurePort=0, ipcPort=45029, storageInfo=lv=-57;cid=testClusterID;nsid=1240587590;c=1731514082621) 2024-11-13T16:08:06,481 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb01705cc8c5692ae with lease ID 0xe17620f2bfeeb2b6: from storage DS-83f68ee5-197b-4527-968f-c0dc23954597 node DatanodeRegistration(127.0.0.1:33087, datanodeUuid=4c2e890c-e34f-48b7-b831-a71ec0b9834d, infoPort=38919, infoSecurePort=0, ipcPort=45029, storageInfo=lv=-57;cid=testClusterID;nsid=1240587590;c=1731514082621), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T16:08:06,481 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb6ab7af76ac8a0ab with lease ID 0xe17620f2bfeeb2b7: Processing first storage report for DS-fd3ddb02-0401-4398-a2bc-f4c7aca6fd63 from datanode DatanodeRegistration(127.0.0.1:45743, datanodeUuid=3de4e64d-0c3f-4dd4-b7a5-e67576d85d5e, infoPort=46455, infoSecurePort=0, ipcPort=33353, storageInfo=lv=-57;cid=testClusterID;nsid=1240587590;c=1731514082621) 2024-11-13T16:08:06,481 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb6ab7af76ac8a0ab with lease ID 0xe17620f2bfeeb2b7: from storage DS-fd3ddb02-0401-4398-a2bc-f4c7aca6fd63 node DatanodeRegistration(127.0.0.1:45743, datanodeUuid=3de4e64d-0c3f-4dd4-b7a5-e67576d85d5e, infoPort=46455, infoSecurePort=0, ipcPort=33353, storageInfo=lv=-57;cid=testClusterID;nsid=1240587590;c=1731514082621), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T16:08:06,481 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x259e61375e0914da with lease ID 0xe17620f2bfeeb2b5: Processing first storage report for DS-cd7dba27-2975-42c9-9bc2-365bf3fcf2d4 from datanode DatanodeRegistration(127.0.0.1:32863, datanodeUuid=fc672ff6-9eb1-4fc8-a112-e265d00093c2, infoPort=33713, infoSecurePort=0, ipcPort=43595, storageInfo=lv=-57;cid=testClusterID;nsid=1240587590;c=1731514082621) 2024-11-13T16:08:06,481 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x259e61375e0914da with lease ID 0xe17620f2bfeeb2b5: from storage DS-cd7dba27-2975-42c9-9bc2-365bf3fcf2d4 node DatanodeRegistration(127.0.0.1:32863, datanodeUuid=fc672ff6-9eb1-4fc8-a112-e265d00093c2, infoPort=33713, infoSecurePort=0, ipcPort=43595, storageInfo=lv=-57;cid=testClusterID;nsid=1240587590;c=1731514082621), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T16:08:06,481 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb01705cc8c5692ae with lease ID 0xe17620f2bfeeb2b6: Processing first storage report for DS-c6c72e8c-d67d-4d4c-9521-1323b5856c0a from datanode DatanodeRegistration(127.0.0.1:33087, datanodeUuid=4c2e890c-e34f-48b7-b831-a71ec0b9834d, infoPort=38919, infoSecurePort=0, ipcPort=45029, storageInfo=lv=-57;cid=testClusterID;nsid=1240587590;c=1731514082621) 2024-11-13T16:08:06,482 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb01705cc8c5692ae with lease ID 0xe17620f2bfeeb2b6: from storage DS-c6c72e8c-d67d-4d4c-9521-1323b5856c0a node DatanodeRegistration(127.0.0.1:33087, datanodeUuid=4c2e890c-e34f-48b7-b831-a71ec0b9834d, infoPort=38919, infoSecurePort=0, ipcPort=45029, storageInfo=lv=-57;cid=testClusterID;nsid=1240587590;c=1731514082621), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T16:08:06,533 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3772cfda-71de-26d3-3229-45fcb377a4d9 2024-11-13T16:08:06,611 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3772cfda-71de-26d3-3229-45fcb377a4d9/cluster_1b15728d-283e-8daa-5d9c-302860a3b764/zookeeper_0, clientPort=60455, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3772cfda-71de-26d3-3229-45fcb377a4d9/cluster_1b15728d-283e-8daa-5d9c-302860a3b764/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3772cfda-71de-26d3-3229-45fcb377a4d9/cluster_1b15728d-283e-8daa-5d9c-302860a3b764/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-13T16:08:06,623 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60455 2024-11-13T16:08:06,633 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T16:08:06,637 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T16:08:06,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741825_1001 (size=7) 2024-11-13T16:08:06,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741825_1001 (size=7) 2024-11-13T16:08:06,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741825_1001 (size=7) 2024-11-13T16:08:07,298 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46 with version=8 2024-11-13T16:08:07,298 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/hbase-staging 2024-11-13T16:08:07,415 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-13T16:08:07,719 INFO [Time-limited test {}] client.ConnectionUtils(128): master/f8f85b0bb1d8:0 server-side Connection retries=45 2024-11-13T16:08:07,732 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T16:08:07,733 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T16:08:07,737 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T16:08:07,738 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T16:08:07,738 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T16:08:07,915 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-13T16:08:07,998 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-13T16:08:08,010 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-13T16:08:08,016 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T16:08:08,045 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 17655 (auto-detected) 2024-11-13T16:08:08,046 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-13T16:08:08,069 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:42949 2024-11-13T16:08:08,092 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42949 connecting to ZooKeeper ensemble=127.0.0.1:60455 2024-11-13T16:08:08,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:429490x0, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T16:08:08,182 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42949-0x1013621fa650000 connected 2024-11-13T16:08:08,357 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T16:08:08,363 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T16:08:08,392 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42949-0x1013621fa650000, quorum=127.0.0.1:60455, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T16:08:08,398 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46, hbase.cluster.distributed=false 2024-11-13T16:08:08,447 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42949-0x1013621fa650000, quorum=127.0.0.1:60455, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T16:08:08,457 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42949 2024-11-13T16:08:08,458 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42949 2024-11-13T16:08:08,463 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42949 2024-11-13T16:08:08,470 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42949 2024-11-13T16:08:08,471 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42949 2024-11-13T16:08:08,601 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f8f85b0bb1d8:0 server-side Connection retries=45 2024-11-13T16:08:08,603 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T16:08:08,604 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T16:08:08,604 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T16:08:08,604 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T16:08:08,604 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T16:08:08,608 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T16:08:08,612 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T16:08:08,614 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:43467 2024-11-13T16:08:08,617 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43467 connecting to ZooKeeper ensemble=127.0.0.1:60455 2024-11-13T16:08:08,618 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T16:08:08,622 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T16:08:08,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:434670x0, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T16:08:08,647 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:434670x0, quorum=127.0.0.1:60455, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T16:08:08,647 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43467-0x1013621fa650001 connected 2024-11-13T16:08:08,653 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T16:08:08,666 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T16:08:08,669 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43467-0x1013621fa650001, quorum=127.0.0.1:60455, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T16:08:08,688 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43467-0x1013621fa650001, quorum=127.0.0.1:60455, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T16:08:08,695 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43467 2024-11-13T16:08:08,698 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43467 2024-11-13T16:08:08,698 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43467 2024-11-13T16:08:08,700 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43467 2024-11-13T16:08:08,700 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43467 2024-11-13T16:08:08,725 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f8f85b0bb1d8:0 server-side Connection retries=45 2024-11-13T16:08:08,726 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T16:08:08,726 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T16:08:08,727 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T16:08:08,727 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T16:08:08,728 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T16:08:08,728 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T16:08:08,729 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T16:08:08,734 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:38187 2024-11-13T16:08:08,737 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38187 connecting to ZooKeeper ensemble=127.0.0.1:60455 2024-11-13T16:08:08,740 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T16:08:08,745 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T16:08:08,771 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:381870x0, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T16:08:08,771 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38187-0x1013621fa650002 connected 2024-11-13T16:08:08,772 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38187-0x1013621fa650002, quorum=127.0.0.1:60455, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T16:08:08,772 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T16:08:08,774 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T16:08:08,775 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38187-0x1013621fa650002, quorum=127.0.0.1:60455, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T16:08:08,778 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38187-0x1013621fa650002, quorum=127.0.0.1:60455, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T16:08:08,779 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38187 2024-11-13T16:08:08,779 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38187 2024-11-13T16:08:08,779 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38187 2024-11-13T16:08:08,780 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38187 2024-11-13T16:08:08,780 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38187 2024-11-13T16:08:08,799 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f8f85b0bb1d8:0 server-side Connection retries=45 2024-11-13T16:08:08,799 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T16:08:08,799 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T16:08:08,800 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T16:08:08,800 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T16:08:08,800 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T16:08:08,801 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T16:08:08,801 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T16:08:08,802 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:41333 2024-11-13T16:08:08,805 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41333 connecting to ZooKeeper ensemble=127.0.0.1:60455 2024-11-13T16:08:08,807 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T16:08:08,811 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T16:08:08,829 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:413330x0, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T16:08:08,830 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41333-0x1013621fa650003 connected 2024-11-13T16:08:08,830 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41333-0x1013621fa650003, quorum=127.0.0.1:60455, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T16:08:08,831 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T16:08:08,836 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T16:08:08,838 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41333-0x1013621fa650003, quorum=127.0.0.1:60455, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T16:08:08,842 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41333-0x1013621fa650003, quorum=127.0.0.1:60455, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T16:08:08,843 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41333 2024-11-13T16:08:08,844 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41333 2024-11-13T16:08:08,844 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41333 2024-11-13T16:08:08,845 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41333 2024-11-13T16:08:08,846 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41333 2024-11-13T16:08:08,865 DEBUG [M:0;f8f85b0bb1d8:42949 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;f8f85b0bb1d8:42949 2024-11-13T16:08:08,866 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/f8f85b0bb1d8,42949,1731514087490 2024-11-13T16:08:08,885 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43467-0x1013621fa650001, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T16:08:08,885 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42949-0x1013621fa650000, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T16:08:08,885 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38187-0x1013621fa650002, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T16:08:08,885 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41333-0x1013621fa650003, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T16:08:08,887 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42949-0x1013621fa650000, quorum=127.0.0.1:60455, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/f8f85b0bb1d8,42949,1731514087490 2024-11-13T16:08:08,918 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41333-0x1013621fa650003, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T16:08:08,918 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38187-0x1013621fa650002, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T16:08:08,918 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41333-0x1013621fa650003, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:08:08,918 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42949-0x1013621fa650000, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:08:08,918 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38187-0x1013621fa650002, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:08:08,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43467-0x1013621fa650001, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T16:08:08,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43467-0x1013621fa650001, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:08:08,920 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42949-0x1013621fa650000, quorum=127.0.0.1:60455, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-13T16:08:08,922 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/f8f85b0bb1d8,42949,1731514087490 from backup master directory 2024-11-13T16:08:08,928 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42949-0x1013621fa650000, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/f8f85b0bb1d8,42949,1731514087490 2024-11-13T16:08:08,928 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38187-0x1013621fa650002, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T16:08:08,928 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43467-0x1013621fa650001, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T16:08:08,928 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41333-0x1013621fa650003, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T16:08:08,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42949-0x1013621fa650000, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T16:08:08,929 WARN [master/f8f85b0bb1d8:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T16:08:08,930 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=f8f85b0bb1d8,42949,1731514087490 2024-11-13T16:08:08,932 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-13T16:08:08,934 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-13T16:08:09,009 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/hbase.id] with ID: 5d0c4b15-0fa1-4b67-8c04-53b67980164f 2024-11-13T16:08:09,009 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/.tmp/hbase.id 2024-11-13T16:08:09,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741826_1002 (size=42) 2024-11-13T16:08:09,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741826_1002 (size=42) 2024-11-13T16:08:09,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741826_1002 (size=42) 2024-11-13T16:08:09,072 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/.tmp/hbase.id]:[hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/hbase.id] 2024-11-13T16:08:09,146 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T16:08:09,152 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-13T16:08:09,174 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 21ms. 2024-11-13T16:08:09,187 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43467-0x1013621fa650001, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:08:09,187 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38187-0x1013621fa650002, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:08:09,187 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41333-0x1013621fa650003, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:08:09,187 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42949-0x1013621fa650000, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:08:09,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741827_1003 (size=196) 2024-11-13T16:08:09,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741827_1003 (size=196) 2024-11-13T16:08:09,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741827_1003 (size=196) 2024-11-13T16:08:09,238 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T16:08:09,242 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-13T16:08:09,267 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:08:09,275 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T16:08:09,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741828_1004 (size=1189) 2024-11-13T16:08:09,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741828_1004 (size=1189) 2024-11-13T16:08:09,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741828_1004 (size=1189) 2024-11-13T16:08:09,370 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/MasterData/data/master/store 2024-11-13T16:08:09,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741829_1005 (size=34) 2024-11-13T16:08:09,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741829_1005 (size=34) 2024-11-13T16:08:09,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741829_1005 (size=34) 2024-11-13T16:08:09,414 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-13T16:08:09,417 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:09,419 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T16:08:09,419 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T16:08:09,420 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T16:08:09,422 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T16:08:09,422 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T16:08:09,422 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T16:08:09,423 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731514089419Disabling compacts and flushes for region at 1731514089419Disabling writes for close at 1731514089422 (+3 ms)Writing region close event to WAL at 1731514089422Closed at 1731514089422 2024-11-13T16:08:09,426 WARN [master/f8f85b0bb1d8:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/MasterData/data/master/store/.initializing 2024-11-13T16:08:09,426 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/MasterData/WALs/f8f85b0bb1d8,42949,1731514087490 2024-11-13T16:08:09,440 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T16:08:09,461 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f8f85b0bb1d8%2C42949%2C1731514087490, suffix=, logDir=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/MasterData/WALs/f8f85b0bb1d8,42949,1731514087490, archiveDir=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/MasterData/oldWALs, maxLogs=10 2024-11-13T16:08:09,492 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/MasterData/WALs/f8f85b0bb1d8,42949,1731514087490/f8f85b0bb1d8%2C42949%2C1731514087490.1731514089466, exclude list is [], retry=0 2024-11-13T16:08:09,515 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33087,DS-83f68ee5-197b-4527-968f-c0dc23954597,DISK] 2024-11-13T16:08:09,515 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32863,DS-af4262a8-b437-4e57-821f-0315c01636b9,DISK] 2024-11-13T16:08:09,515 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45743,DS-b848e0f5-c5b4-4830-9d9c-fc9502100de9,DISK] 2024-11-13T16:08:09,519 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-13T16:08:09,579 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/MasterData/WALs/f8f85b0bb1d8,42949,1731514087490/f8f85b0bb1d8%2C42949%2C1731514087490.1731514089466 2024-11-13T16:08:09,581 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33713:33713),(127.0.0.1/127.0.0.1:38919:38919),(127.0.0.1/127.0.0.1:46455:46455)] 2024-11-13T16:08:09,582 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-13T16:08:09,582 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:09,587 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T16:08:09,588 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T16:08:09,630 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T16:08:09,663 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-13T16:08:09,667 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:09,670 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T16:08:09,671 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T16:08:09,676 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-13T16:08:09,676 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:09,678 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:09,678 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T16:08:09,682 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-13T16:08:09,682 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:09,684 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:09,684 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T16:08:09,688 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-13T16:08:09,688 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:09,689 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:09,690 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T16:08:09,695 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T16:08:09,697 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T16:08:09,704 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T16:08:09,705 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T16:08:09,709 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T16:08:09,715 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T16:08:09,722 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T16:08:09,724 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67131441, jitterRate=3.3642351627349854E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T16:08:09,732 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731514089602Initializing all the Stores at 1731514089605 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731514089606 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514089608 (+2 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514089609 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514089609Cleaning up temporary data from old regions at 1731514089705 (+96 ms)Region opened successfully at 1731514089732 (+27 ms) 2024-11-13T16:08:09,734 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-13T16:08:09,783 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@747a1ea1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f8f85b0bb1d8/172.17.0.3:0 2024-11-13T16:08:09,824 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-13T16:08:09,839 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-13T16:08:09,839 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-13T16:08:09,843 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T16:08:09,845 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-13T16:08:09,850 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-13T16:08:09,850 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-13T16:08:09,883 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-13T16:08:09,895 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42949-0x1013621fa650000, quorum=127.0.0.1:60455, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-13T16:08:09,907 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-13T16:08:09,910 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-13T16:08:09,912 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42949-0x1013621fa650000, quorum=127.0.0.1:60455, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-13T16:08:09,920 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-13T16:08:09,923 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-13T16:08:09,927 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42949-0x1013621fa650000, quorum=127.0.0.1:60455, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-13T16:08:09,936 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-13T16:08:09,938 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42949-0x1013621fa650000, quorum=127.0.0.1:60455, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-13T16:08:09,945 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-13T16:08:09,963 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42949-0x1013621fa650000, quorum=127.0.0.1:60455, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-13T16:08:09,970 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-13T16:08:09,984 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43467-0x1013621fa650001, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T16:08:09,984 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41333-0x1013621fa650003, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T16:08:09,984 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42949-0x1013621fa650000, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T16:08:09,984 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38187-0x1013621fa650002, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T16:08:09,984 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43467-0x1013621fa650001, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:08:09,984 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42949-0x1013621fa650000, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:08:09,984 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41333-0x1013621fa650003, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:08:09,984 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38187-0x1013621fa650002, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:08:09,988 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=f8f85b0bb1d8,42949,1731514087490, sessionid=0x1013621fa650000, setting cluster-up flag (Was=false) 2024-11-13T16:08:10,017 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38187-0x1013621fa650002, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:08:10,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41333-0x1013621fa650003, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:08:10,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43467-0x1013621fa650001, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:08:10,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42949-0x1013621fa650000, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:08:10,045 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-13T16:08:10,049 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f8f85b0bb1d8,42949,1731514087490 2024-11-13T16:08:10,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42949-0x1013621fa650000, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:08:10,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41333-0x1013621fa650003, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:08:10,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43467-0x1013621fa650001, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:08:10,070 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38187-0x1013621fa650002, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:08:10,095 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-13T16:08:10,098 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f8f85b0bb1d8,42949,1731514087490 2024-11-13T16:08:10,105 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-13T16:08:10,151 INFO [RS:1;f8f85b0bb1d8:38187 {}] regionserver.HRegionServer(746): ClusterId : 5d0c4b15-0fa1-4b67-8c04-53b67980164f 2024-11-13T16:08:10,151 INFO [RS:0;f8f85b0bb1d8:43467 {}] regionserver.HRegionServer(746): ClusterId : 5d0c4b15-0fa1-4b67-8c04-53b67980164f 2024-11-13T16:08:10,151 INFO [RS:2;f8f85b0bb1d8:41333 {}] regionserver.HRegionServer(746): ClusterId : 5d0c4b15-0fa1-4b67-8c04-53b67980164f 2024-11-13T16:08:10,153 DEBUG [RS:0;f8f85b0bb1d8:43467 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T16:08:10,153 DEBUG [RS:2;f8f85b0bb1d8:41333 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T16:08:10,153 DEBUG [RS:1;f8f85b0bb1d8:38187 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T16:08:10,172 DEBUG [RS:1;f8f85b0bb1d8:38187 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T16:08:10,172 DEBUG [RS:0;f8f85b0bb1d8:43467 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T16:08:10,172 DEBUG [RS:2;f8f85b0bb1d8:41333 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T16:08:10,172 DEBUG [RS:1;f8f85b0bb1d8:38187 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T16:08:10,172 DEBUG [RS:2;f8f85b0bb1d8:41333 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T16:08:10,172 DEBUG [RS:0;f8f85b0bb1d8:43467 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T16:08:10,189 DEBUG [RS:2;f8f85b0bb1d8:41333 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T16:08:10,189 DEBUG [RS:1;f8f85b0bb1d8:38187 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T16:08:10,190 DEBUG [RS:1;f8f85b0bb1d8:38187 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79ec3021, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f8f85b0bb1d8/172.17.0.3:0 2024-11-13T16:08:10,190 DEBUG [RS:0;f8f85b0bb1d8:43467 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T16:08:10,190 DEBUG [RS:2;f8f85b0bb1d8:41333 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@94f9856, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f8f85b0bb1d8/172.17.0.3:0 2024-11-13T16:08:10,191 DEBUG [RS:0;f8f85b0bb1d8:43467 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40112172, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f8f85b0bb1d8/172.17.0.3:0 2024-11-13T16:08:10,200 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T16:08:10,215 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-13T16:08:10,217 DEBUG [RS:1;f8f85b0bb1d8:38187 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;f8f85b0bb1d8:38187 2024-11-13T16:08:10,222 INFO [RS:1;f8f85b0bb1d8:38187 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T16:08:10,223 INFO [RS:1;f8f85b0bb1d8:38187 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T16:08:10,223 DEBUG [RS:1;f8f85b0bb1d8:38187 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T16:08:10,228 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-13T16:08:10,232 INFO [RS:1;f8f85b0bb1d8:38187 {}] regionserver.HRegionServer(2659): reportForDuty to master=f8f85b0bb1d8,42949,1731514087490 with port=38187, startcode=1731514088725 2024-11-13T16:08:10,232 DEBUG [RS:2;f8f85b0bb1d8:41333 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;f8f85b0bb1d8:41333 2024-11-13T16:08:10,233 INFO [RS:2;f8f85b0bb1d8:41333 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T16:08:10,233 INFO [RS:2;f8f85b0bb1d8:41333 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T16:08:10,233 DEBUG [RS:2;f8f85b0bb1d8:41333 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T16:08:10,240 DEBUG [RS:0;f8f85b0bb1d8:43467 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;f8f85b0bb1d8:43467 2024-11-13T16:08:10,240 INFO [RS:0;f8f85b0bb1d8:43467 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T16:08:10,240 INFO [RS:0;f8f85b0bb1d8:43467 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T16:08:10,240 DEBUG [RS:0;f8f85b0bb1d8:43467 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T16:08:10,243 INFO [RS:2;f8f85b0bb1d8:41333 {}] regionserver.HRegionServer(2659): reportForDuty to master=f8f85b0bb1d8,42949,1731514087490 with port=41333, startcode=1731514088798 2024-11-13T16:08:10,240 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: f8f85b0bb1d8,42949,1731514087490 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-13T16:08:10,245 INFO [RS:0;f8f85b0bb1d8:43467 {}] regionserver.HRegionServer(2659): reportForDuty to master=f8f85b0bb1d8,42949,1731514087490 with port=43467, startcode=1731514088560 2024-11-13T16:08:10,252 DEBUG [RS:2;f8f85b0bb1d8:41333 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T16:08:10,252 DEBUG [RS:0;f8f85b0bb1d8:43467 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T16:08:10,252 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/f8f85b0bb1d8:0, corePoolSize=5, maxPoolSize=5 2024-11-13T16:08:10,252 DEBUG [RS:1;f8f85b0bb1d8:38187 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T16:08:10,252 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/f8f85b0bb1d8:0, corePoolSize=5, maxPoolSize=5 2024-11-13T16:08:10,252 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/f8f85b0bb1d8:0, corePoolSize=5, maxPoolSize=5 2024-11-13T16:08:10,252 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/f8f85b0bb1d8:0, corePoolSize=5, maxPoolSize=5 2024-11-13T16:08:10,253 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/f8f85b0bb1d8:0, corePoolSize=10, maxPoolSize=10 2024-11-13T16:08:10,253 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:08:10,253 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/f8f85b0bb1d8:0, corePoolSize=2, maxPoolSize=2 2024-11-13T16:08:10,253 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:08:10,297 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731514120297 2024-11-13T16:08:10,302 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-13T16:08:10,303 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-13T16:08:10,307 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-13T16:08:10,308 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-13T16:08:10,308 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-13T16:08:10,308 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-13T16:08:10,312 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T16:08:10,313 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-13T16:08:10,313 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T16:08:10,319 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:10,319 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-13T16:08:10,324 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-13T16:08:10,326 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-13T16:08:10,326 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-13T16:08:10,335 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-13T16:08:10,336 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-13T16:08:10,337 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51581, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T16:08:10,337 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36241, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T16:08:10,339 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32773, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T16:08:10,345 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42949 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T16:08:10,351 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/f8f85b0bb1d8:0:becomeActiveMaster-HFileCleaner.large.0-1731514090338,5,FailOnTimeoutGroup] 2024-11-13T16:08:10,353 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42949 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T16:08:10,354 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42949 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T16:08:10,363 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/f8f85b0bb1d8:0:becomeActiveMaster-HFileCleaner.small.0-1731514090351,5,FailOnTimeoutGroup] 2024-11-13T16:08:10,363 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T16:08:10,363 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-13T16:08:10,364 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-13T16:08:10,365 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-13T16:08:10,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741831_1007 (size=1321) 2024-11-13T16:08:10,404 DEBUG [RS:1;f8f85b0bb1d8:38187 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-13T16:08:10,404 WARN [RS:1;f8f85b0bb1d8:38187 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-13T16:08:10,405 DEBUG [RS:0;f8f85b0bb1d8:43467 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-13T16:08:10,405 WARN [RS:0;f8f85b0bb1d8:43467 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-13T16:08:10,405 DEBUG [RS:2;f8f85b0bb1d8:41333 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-13T16:08:10,405 WARN [RS:2;f8f85b0bb1d8:41333 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-13T16:08:10,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741831_1007 (size=1321) 2024-11-13T16:08:10,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741831_1007 (size=1321) 2024-11-13T16:08:10,408 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-13T16:08:10,408 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46 2024-11-13T16:08:10,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741832_1008 (size=32) 2024-11-13T16:08:10,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741832_1008 (size=32) 2024-11-13T16:08:10,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741832_1008 (size=32) 2024-11-13T16:08:10,440 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:10,444 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T16:08:10,448 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T16:08:10,448 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:10,449 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T16:08:10,450 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T16:08:10,453 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T16:08:10,453 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:10,454 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T16:08:10,454 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T16:08:10,457 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T16:08:10,457 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:10,458 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T16:08:10,459 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T16:08:10,462 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T16:08:10,462 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:10,463 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T16:08:10,463 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T16:08:10,465 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/hbase/meta/1588230740 2024-11-13T16:08:10,466 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/hbase/meta/1588230740 2024-11-13T16:08:10,470 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T16:08:10,470 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T16:08:10,471 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T16:08:10,475 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T16:08:10,480 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T16:08:10,481 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66926663, jitterRate=-0.002715006470680237}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T16:08:10,485 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731514090440Initializing all the Stores at 1731514090442 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731514090442Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731514090444 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514090444Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731514090444Cleaning up temporary data from old regions at 1731514090470 (+26 ms)Region opened successfully at 1731514090485 (+15 ms) 2024-11-13T16:08:10,486 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T16:08:10,486 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T16:08:10,486 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T16:08:10,486 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T16:08:10,486 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T16:08:10,488 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T16:08:10,489 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731514090485Disabling compacts and flushes for region at 1731514090486 (+1 ms)Disabling writes for close at 1731514090486Writing region close event to WAL at 1731514090487 (+1 ms)Closed at 1731514090488 (+1 ms) 2024-11-13T16:08:10,492 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T16:08:10,492 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-13T16:08:10,500 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-13T16:08:10,506 INFO [RS:1;f8f85b0bb1d8:38187 {}] regionserver.HRegionServer(2659): reportForDuty to master=f8f85b0bb1d8,42949,1731514087490 with port=38187, startcode=1731514088725 2024-11-13T16:08:10,507 INFO [RS:0;f8f85b0bb1d8:43467 {}] regionserver.HRegionServer(2659): reportForDuty to master=f8f85b0bb1d8,42949,1731514087490 with port=43467, startcode=1731514088560 2024-11-13T16:08:10,507 INFO [RS:2;f8f85b0bb1d8:41333 {}] regionserver.HRegionServer(2659): reportForDuty to master=f8f85b0bb1d8,42949,1731514087490 with port=41333, startcode=1731514088798 2024-11-13T16:08:10,509 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42949 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:08:10,512 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42949 {}] master.ServerManager(517): Registering regionserver=f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:08:10,515 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T16:08:10,519 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-13T16:08:10,521 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42949 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f8f85b0bb1d8,43467,1731514088560 2024-11-13T16:08:10,521 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42949 {}] master.ServerManager(517): Registering regionserver=f8f85b0bb1d8,43467,1731514088560 2024-11-13T16:08:10,522 DEBUG [RS:2;f8f85b0bb1d8:41333 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46 2024-11-13T16:08:10,522 DEBUG [RS:2;f8f85b0bb1d8:41333 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44161 2024-11-13T16:08:10,522 DEBUG [RS:2;f8f85b0bb1d8:41333 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T16:08:10,525 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42949 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f8f85b0bb1d8,38187,1731514088725 2024-11-13T16:08:10,525 DEBUG [RS:0;f8f85b0bb1d8:43467 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46 2024-11-13T16:08:10,525 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42949 {}] master.ServerManager(517): Registering regionserver=f8f85b0bb1d8,38187,1731514088725 2024-11-13T16:08:10,525 DEBUG [RS:0;f8f85b0bb1d8:43467 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44161 2024-11-13T16:08:10,525 DEBUG [RS:0;f8f85b0bb1d8:43467 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T16:08:10,529 DEBUG [RS:1;f8f85b0bb1d8:38187 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46 2024-11-13T16:08:10,529 DEBUG [RS:1;f8f85b0bb1d8:38187 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44161 2024-11-13T16:08:10,529 DEBUG [RS:1;f8f85b0bb1d8:38187 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T16:08:10,534 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42949-0x1013621fa650000, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T16:08:10,582 DEBUG [RS:2;f8f85b0bb1d8:41333 {}] zookeeper.ZKUtil(111): regionserver:41333-0x1013621fa650003, quorum=127.0.0.1:60455, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:08:10,582 WARN [RS:2;f8f85b0bb1d8:41333 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T16:08:10,583 DEBUG [RS:1;f8f85b0bb1d8:38187 {}] zookeeper.ZKUtil(111): regionserver:38187-0x1013621fa650002, quorum=127.0.0.1:60455, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f8f85b0bb1d8,38187,1731514088725 2024-11-13T16:08:10,583 DEBUG [RS:0;f8f85b0bb1d8:43467 {}] zookeeper.ZKUtil(111): regionserver:43467-0x1013621fa650001, quorum=127.0.0.1:60455, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f8f85b0bb1d8,43467,1731514088560 2024-11-13T16:08:10,583 INFO [RS:2;f8f85b0bb1d8:41333 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T16:08:10,583 WARN [RS:1;f8f85b0bb1d8:38187 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T16:08:10,583 WARN [RS:0;f8f85b0bb1d8:43467 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T16:08:10,583 INFO [RS:1;f8f85b0bb1d8:38187 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T16:08:10,583 DEBUG [RS:2;f8f85b0bb1d8:41333 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/WALs/f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:08:10,583 INFO [RS:0;f8f85b0bb1d8:43467 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T16:08:10,583 DEBUG [RS:1;f8f85b0bb1d8:38187 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/WALs/f8f85b0bb1d8,38187,1731514088725 2024-11-13T16:08:10,583 DEBUG [RS:0;f8f85b0bb1d8:43467 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/WALs/f8f85b0bb1d8,43467,1731514088560 2024-11-13T16:08:10,584 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f8f85b0bb1d8,43467,1731514088560] 2024-11-13T16:08:10,584 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f8f85b0bb1d8,41333,1731514088798] 2024-11-13T16:08:10,584 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f8f85b0bb1d8,38187,1731514088725] 2024-11-13T16:08:10,613 INFO [RS:0;f8f85b0bb1d8:43467 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T16:08:10,613 INFO [RS:2;f8f85b0bb1d8:41333 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T16:08:10,613 INFO [RS:1;f8f85b0bb1d8:38187 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T16:08:10,635 INFO [RS:0;f8f85b0bb1d8:43467 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T16:08:10,639 INFO [RS:1;f8f85b0bb1d8:38187 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T16:08:10,639 INFO [RS:2;f8f85b0bb1d8:41333 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T16:08:10,652 INFO [RS:0;f8f85b0bb1d8:43467 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T16:08:10,652 INFO [RS:0;f8f85b0bb1d8:43467 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T16:08:10,652 INFO [RS:1;f8f85b0bb1d8:38187 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T16:08:10,652 INFO [RS:1;f8f85b0bb1d8:38187 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T16:08:10,653 INFO [RS:2;f8f85b0bb1d8:41333 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T16:08:10,653 INFO [RS:2;f8f85b0bb1d8:41333 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T16:08:10,655 INFO [RS:0;f8f85b0bb1d8:43467 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T16:08:10,656 INFO [RS:1;f8f85b0bb1d8:38187 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T16:08:10,657 INFO [RS:2;f8f85b0bb1d8:41333 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T16:08:10,667 INFO [RS:0;f8f85b0bb1d8:43467 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T16:08:10,667 INFO [RS:1;f8f85b0bb1d8:38187 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T16:08:10,668 INFO [RS:2;f8f85b0bb1d8:41333 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T16:08:10,669 INFO [RS:2;f8f85b0bb1d8:41333 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T16:08:10,669 INFO [RS:0;f8f85b0bb1d8:43467 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T16:08:10,669 INFO [RS:1;f8f85b0bb1d8:38187 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T16:08:10,670 DEBUG [RS:0;f8f85b0bb1d8:43467 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:08:10,670 DEBUG [RS:1;f8f85b0bb1d8:38187 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:08:10,670 DEBUG [RS:2;f8f85b0bb1d8:41333 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:08:10,670 DEBUG [RS:0;f8f85b0bb1d8:43467 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:08:10,670 DEBUG [RS:1;f8f85b0bb1d8:38187 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:08:10,670 DEBUG [RS:2;f8f85b0bb1d8:41333 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:08:10,670 DEBUG [RS:0;f8f85b0bb1d8:43467 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:08:10,670 DEBUG [RS:1;f8f85b0bb1d8:38187 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:08:10,670 DEBUG [RS:2;f8f85b0bb1d8:41333 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:08:10,670 DEBUG [RS:1;f8f85b0bb1d8:38187 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:08:10,670 DEBUG [RS:2;f8f85b0bb1d8:41333 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:08:10,670 WARN [f8f85b0bb1d8:42949 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-13T16:08:10,670 DEBUG [RS:0;f8f85b0bb1d8:43467 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:08:10,670 DEBUG [RS:2;f8f85b0bb1d8:41333 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:08:10,670 DEBUG [RS:1;f8f85b0bb1d8:38187 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:08:10,671 DEBUG [RS:0;f8f85b0bb1d8:43467 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:08:10,671 DEBUG [RS:2;f8f85b0bb1d8:41333 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0, corePoolSize=2, maxPoolSize=2 2024-11-13T16:08:10,671 DEBUG [RS:1;f8f85b0bb1d8:38187 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0, corePoolSize=2, maxPoolSize=2 2024-11-13T16:08:10,671 DEBUG [RS:0;f8f85b0bb1d8:43467 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0, corePoolSize=2, maxPoolSize=2 2024-11-13T16:08:10,671 DEBUG [RS:2;f8f85b0bb1d8:41333 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:08:10,671 DEBUG [RS:1;f8f85b0bb1d8:38187 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:08:10,671 DEBUG [RS:0;f8f85b0bb1d8:43467 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:08:10,671 DEBUG [RS:1;f8f85b0bb1d8:38187 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:08:10,671 DEBUG [RS:2;f8f85b0bb1d8:41333 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:08:10,671 DEBUG [RS:1;f8f85b0bb1d8:38187 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:08:10,671 DEBUG [RS:0;f8f85b0bb1d8:43467 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:08:10,671 DEBUG [RS:2;f8f85b0bb1d8:41333 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:08:10,671 DEBUG [RS:1;f8f85b0bb1d8:38187 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:08:10,671 DEBUG [RS:0;f8f85b0bb1d8:43467 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:08:10,671 DEBUG [RS:2;f8f85b0bb1d8:41333 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:08:10,671 DEBUG [RS:1;f8f85b0bb1d8:38187 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:08:10,671 DEBUG [RS:2;f8f85b0bb1d8:41333 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:08:10,671 DEBUG [RS:0;f8f85b0bb1d8:43467 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:08:10,671 DEBUG [RS:1;f8f85b0bb1d8:38187 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:08:10,671 DEBUG [RS:2;f8f85b0bb1d8:41333 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:08:10,671 DEBUG [RS:0;f8f85b0bb1d8:43467 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:08:10,672 DEBUG [RS:1;f8f85b0bb1d8:38187 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f8f85b0bb1d8:0, corePoolSize=3, maxPoolSize=3 2024-11-13T16:08:10,672 DEBUG [RS:2;f8f85b0bb1d8:41333 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f8f85b0bb1d8:0, corePoolSize=3, maxPoolSize=3 2024-11-13T16:08:10,672 DEBUG [RS:0;f8f85b0bb1d8:43467 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:08:10,672 DEBUG [RS:1;f8f85b0bb1d8:38187 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f8f85b0bb1d8:0, corePoolSize=3, maxPoolSize=3 2024-11-13T16:08:10,672 DEBUG [RS:2;f8f85b0bb1d8:41333 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f8f85b0bb1d8:0, corePoolSize=3, maxPoolSize=3 2024-11-13T16:08:10,672 DEBUG [RS:0;f8f85b0bb1d8:43467 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f8f85b0bb1d8:0, corePoolSize=3, maxPoolSize=3 2024-11-13T16:08:10,672 DEBUG [RS:0;f8f85b0bb1d8:43467 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f8f85b0bb1d8:0, corePoolSize=3, maxPoolSize=3 2024-11-13T16:08:10,684 INFO [RS:2;f8f85b0bb1d8:41333 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T16:08:10,684 INFO [RS:1;f8f85b0bb1d8:38187 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T16:08:10,684 INFO [RS:0;f8f85b0bb1d8:43467 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T16:08:10,684 INFO [RS:1;f8f85b0bb1d8:38187 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T16:08:10,684 INFO [RS:0;f8f85b0bb1d8:43467 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T16:08:10,684 INFO [RS:2;f8f85b0bb1d8:41333 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T16:08:10,685 INFO [RS:0;f8f85b0bb1d8:43467 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T16:08:10,685 INFO [RS:1;f8f85b0bb1d8:38187 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T16:08:10,685 INFO [RS:2;f8f85b0bb1d8:41333 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T16:08:10,685 INFO [RS:1;f8f85b0bb1d8:38187 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T16:08:10,685 INFO [RS:0;f8f85b0bb1d8:43467 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T16:08:10,685 INFO [RS:2;f8f85b0bb1d8:41333 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T16:08:10,685 INFO [RS:0;f8f85b0bb1d8:43467 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T16:08:10,685 INFO [RS:1;f8f85b0bb1d8:38187 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T16:08:10,685 INFO [RS:2;f8f85b0bb1d8:41333 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T16:08:10,685 INFO [RS:1;f8f85b0bb1d8:38187 {}] hbase.ChoreService(168): Chore ScheduledChore name=f8f85b0bb1d8,38187,1731514088725-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T16:08:10,685 INFO [RS:0;f8f85b0bb1d8:43467 {}] hbase.ChoreService(168): Chore ScheduledChore name=f8f85b0bb1d8,43467,1731514088560-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T16:08:10,685 INFO [RS:2;f8f85b0bb1d8:41333 {}] hbase.ChoreService(168): Chore ScheduledChore name=f8f85b0bb1d8,41333,1731514088798-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T16:08:10,712 INFO [RS:2;f8f85b0bb1d8:41333 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T16:08:10,712 INFO [RS:0;f8f85b0bb1d8:43467 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T16:08:10,714 INFO [RS:0;f8f85b0bb1d8:43467 {}] hbase.ChoreService(168): Chore ScheduledChore name=f8f85b0bb1d8,43467,1731514088560-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T16:08:10,715 INFO [RS:0;f8f85b0bb1d8:43467 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T16:08:10,715 INFO [RS:2;f8f85b0bb1d8:41333 {}] hbase.ChoreService(168): Chore ScheduledChore name=f8f85b0bb1d8,41333,1731514088798-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T16:08:10,715 INFO [RS:0;f8f85b0bb1d8:43467 {}] regionserver.Replication(171): f8f85b0bb1d8,43467,1731514088560 started 2024-11-13T16:08:10,715 INFO [RS:2;f8f85b0bb1d8:41333 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T16:08:10,715 INFO [RS:2;f8f85b0bb1d8:41333 {}] regionserver.Replication(171): f8f85b0bb1d8,41333,1731514088798 started 2024-11-13T16:08:10,716 INFO [RS:1;f8f85b0bb1d8:38187 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T16:08:10,717 INFO [RS:1;f8f85b0bb1d8:38187 {}] hbase.ChoreService(168): Chore ScheduledChore name=f8f85b0bb1d8,38187,1731514088725-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T16:08:10,717 INFO [RS:1;f8f85b0bb1d8:38187 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T16:08:10,717 INFO [RS:1;f8f85b0bb1d8:38187 {}] regionserver.Replication(171): f8f85b0bb1d8,38187,1731514088725 started 2024-11-13T16:08:10,737 INFO [RS:1;f8f85b0bb1d8:38187 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T16:08:10,738 INFO [RS:1;f8f85b0bb1d8:38187 {}] regionserver.HRegionServer(1482): Serving as f8f85b0bb1d8,38187,1731514088725, RpcServer on f8f85b0bb1d8/172.17.0.3:38187, sessionid=0x1013621fa650002 2024-11-13T16:08:10,739 DEBUG [RS:1;f8f85b0bb1d8:38187 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T16:08:10,739 DEBUG [RS:1;f8f85b0bb1d8:38187 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f8f85b0bb1d8,38187,1731514088725 2024-11-13T16:08:10,739 DEBUG [RS:1;f8f85b0bb1d8:38187 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f8f85b0bb1d8,38187,1731514088725' 2024-11-13T16:08:10,739 INFO [RS:0;f8f85b0bb1d8:43467 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T16:08:10,739 DEBUG [RS:1;f8f85b0bb1d8:38187 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T16:08:10,739 INFO [RS:0;f8f85b0bb1d8:43467 {}] regionserver.HRegionServer(1482): Serving as f8f85b0bb1d8,43467,1731514088560, RpcServer on f8f85b0bb1d8/172.17.0.3:43467, sessionid=0x1013621fa650001 2024-11-13T16:08:10,740 DEBUG [RS:0;f8f85b0bb1d8:43467 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T16:08:10,740 DEBUG [RS:0;f8f85b0bb1d8:43467 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f8f85b0bb1d8,43467,1731514088560 2024-11-13T16:08:10,740 DEBUG [RS:0;f8f85b0bb1d8:43467 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f8f85b0bb1d8,43467,1731514088560' 2024-11-13T16:08:10,740 DEBUG [RS:0;f8f85b0bb1d8:43467 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T16:08:10,740 INFO [RS:2;f8f85b0bb1d8:41333 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T16:08:10,740 INFO [RS:2;f8f85b0bb1d8:41333 {}] regionserver.HRegionServer(1482): Serving as f8f85b0bb1d8,41333,1731514088798, RpcServer on f8f85b0bb1d8/172.17.0.3:41333, sessionid=0x1013621fa650003 2024-11-13T16:08:10,740 DEBUG [RS:2;f8f85b0bb1d8:41333 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T16:08:10,741 DEBUG [RS:2;f8f85b0bb1d8:41333 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:08:10,741 DEBUG [RS:2;f8f85b0bb1d8:41333 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f8f85b0bb1d8,41333,1731514088798' 2024-11-13T16:08:10,741 DEBUG [RS:2;f8f85b0bb1d8:41333 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T16:08:10,741 DEBUG [RS:0;f8f85b0bb1d8:43467 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T16:08:10,741 DEBUG [RS:1;f8f85b0bb1d8:38187 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T16:08:10,741 DEBUG [RS:2;f8f85b0bb1d8:41333 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T16:08:10,742 DEBUG [RS:0;f8f85b0bb1d8:43467 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T16:08:10,742 DEBUG [RS:0;f8f85b0bb1d8:43467 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T16:08:10,742 DEBUG [RS:1;f8f85b0bb1d8:38187 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T16:08:10,742 DEBUG [RS:1;f8f85b0bb1d8:38187 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T16:08:10,742 DEBUG [RS:2;f8f85b0bb1d8:41333 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T16:08:10,742 DEBUG [RS:1;f8f85b0bb1d8:38187 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f8f85b0bb1d8,38187,1731514088725 2024-11-13T16:08:10,742 DEBUG [RS:0;f8f85b0bb1d8:43467 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f8f85b0bb1d8,43467,1731514088560 2024-11-13T16:08:10,742 DEBUG [RS:2;f8f85b0bb1d8:41333 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T16:08:10,742 DEBUG [RS:1;f8f85b0bb1d8:38187 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f8f85b0bb1d8,38187,1731514088725' 2024-11-13T16:08:10,742 DEBUG [RS:0;f8f85b0bb1d8:43467 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f8f85b0bb1d8,43467,1731514088560' 2024-11-13T16:08:10,742 DEBUG [RS:2;f8f85b0bb1d8:41333 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:08:10,742 DEBUG [RS:1;f8f85b0bb1d8:38187 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T16:08:10,742 DEBUG [RS:0;f8f85b0bb1d8:43467 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T16:08:10,743 DEBUG [RS:2;f8f85b0bb1d8:41333 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f8f85b0bb1d8,41333,1731514088798' 2024-11-13T16:08:10,743 DEBUG [RS:2;f8f85b0bb1d8:41333 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T16:08:10,743 DEBUG [RS:1;f8f85b0bb1d8:38187 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T16:08:10,743 DEBUG [RS:0;f8f85b0bb1d8:43467 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T16:08:10,744 DEBUG [RS:2;f8f85b0bb1d8:41333 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T16:08:10,744 DEBUG [RS:1;f8f85b0bb1d8:38187 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T16:08:10,744 INFO [RS:1;f8f85b0bb1d8:38187 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T16:08:10,744 INFO [RS:1;f8f85b0bb1d8:38187 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T16:08:10,745 DEBUG [RS:0;f8f85b0bb1d8:43467 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T16:08:10,745 DEBUG [RS:2;f8f85b0bb1d8:41333 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T16:08:10,745 INFO [RS:0;f8f85b0bb1d8:43467 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T16:08:10,745 INFO [RS:2;f8f85b0bb1d8:41333 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T16:08:10,745 INFO [RS:0;f8f85b0bb1d8:43467 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T16:08:10,745 INFO [RS:2;f8f85b0bb1d8:41333 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T16:08:10,849 INFO [RS:2;f8f85b0bb1d8:41333 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T16:08:10,849 INFO [RS:0;f8f85b0bb1d8:43467 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T16:08:10,849 INFO [RS:1;f8f85b0bb1d8:38187 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T16:08:10,852 INFO [RS:0;f8f85b0bb1d8:43467 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f8f85b0bb1d8%2C43467%2C1731514088560, suffix=, logDir=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/WALs/f8f85b0bb1d8,43467,1731514088560, archiveDir=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/oldWALs, maxLogs=32 2024-11-13T16:08:10,852 INFO [RS:1;f8f85b0bb1d8:38187 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f8f85b0bb1d8%2C38187%2C1731514088725, suffix=, logDir=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/WALs/f8f85b0bb1d8,38187,1731514088725, archiveDir=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/oldWALs, maxLogs=32 2024-11-13T16:08:10,852 INFO [RS:2;f8f85b0bb1d8:41333 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f8f85b0bb1d8%2C41333%2C1731514088798, suffix=, logDir=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/WALs/f8f85b0bb1d8,41333,1731514088798, archiveDir=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/oldWALs, maxLogs=32 2024-11-13T16:08:10,871 DEBUG [RS:0;f8f85b0bb1d8:43467 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/WALs/f8f85b0bb1d8,43467,1731514088560/f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856, exclude list is [], retry=0 2024-11-13T16:08:10,871 DEBUG [RS:2;f8f85b0bb1d8:41333 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/WALs/f8f85b0bb1d8,41333,1731514088798/f8f85b0bb1d8%2C41333%2C1731514088798.1731514090856, exclude list is [], retry=0 2024-11-13T16:08:10,873 DEBUG [RS:1;f8f85b0bb1d8:38187 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/WALs/f8f85b0bb1d8,38187,1731514088725/f8f85b0bb1d8%2C38187%2C1731514088725.1731514090856, exclude list is [], retry=0 2024-11-13T16:08:10,879 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32863,DS-af4262a8-b437-4e57-821f-0315c01636b9,DISK] 2024-11-13T16:08:10,881 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33087,DS-83f68ee5-197b-4527-968f-c0dc23954597,DISK] 2024-11-13T16:08:10,881 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45743,DS-b848e0f5-c5b4-4830-9d9c-fc9502100de9,DISK] 2024-11-13T16:08:10,882 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32863,DS-af4262a8-b437-4e57-821f-0315c01636b9,DISK] 2024-11-13T16:08:10,883 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33087,DS-83f68ee5-197b-4527-968f-c0dc23954597,DISK] 2024-11-13T16:08:10,883 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45743,DS-b848e0f5-c5b4-4830-9d9c-fc9502100de9,DISK] 2024-11-13T16:08:10,884 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33087,DS-83f68ee5-197b-4527-968f-c0dc23954597,DISK] 2024-11-13T16:08:10,884 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32863,DS-af4262a8-b437-4e57-821f-0315c01636b9,DISK] 2024-11-13T16:08:10,884 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45743,DS-b848e0f5-c5b4-4830-9d9c-fc9502100de9,DISK] 2024-11-13T16:08:10,898 INFO [RS:0;f8f85b0bb1d8:43467 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/WALs/f8f85b0bb1d8,43467,1731514088560/f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856 2024-11-13T16:08:10,899 DEBUG [RS:0;f8f85b0bb1d8:43467 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46455:46455),(127.0.0.1/127.0.0.1:38919:38919),(127.0.0.1/127.0.0.1:33713:33713)] 2024-11-13T16:08:10,900 INFO [RS:2;f8f85b0bb1d8:41333 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/WALs/f8f85b0bb1d8,41333,1731514088798/f8f85b0bb1d8%2C41333%2C1731514088798.1731514090856 2024-11-13T16:08:10,900 DEBUG [RS:2;f8f85b0bb1d8:41333 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33713:33713),(127.0.0.1/127.0.0.1:38919:38919),(127.0.0.1/127.0.0.1:46455:46455)] 2024-11-13T16:08:10,903 INFO [RS:1;f8f85b0bb1d8:38187 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/WALs/f8f85b0bb1d8,38187,1731514088725/f8f85b0bb1d8%2C38187%2C1731514088725.1731514090856 2024-11-13T16:08:10,904 DEBUG [RS:1;f8f85b0bb1d8:38187 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33713:33713),(127.0.0.1/127.0.0.1:38919:38919),(127.0.0.1/127.0.0.1:46455:46455)] 2024-11-13T16:08:10,922 DEBUG [f8f85b0bb1d8:42949 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-13T16:08:10,929 DEBUG [f8f85b0bb1d8:42949 {}] balancer.BalancerClusterState(204): Hosts are {f8f85b0bb1d8=0} racks are {/default-rack=0} 2024-11-13T16:08:10,935 DEBUG [f8f85b0bb1d8:42949 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T16:08:10,935 DEBUG [f8f85b0bb1d8:42949 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T16:08:10,936 DEBUG [f8f85b0bb1d8:42949 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-13T16:08:10,936 DEBUG [f8f85b0bb1d8:42949 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T16:08:10,936 DEBUG [f8f85b0bb1d8:42949 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T16:08:10,936 DEBUG [f8f85b0bb1d8:42949 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-13T16:08:10,936 INFO [f8f85b0bb1d8:42949 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T16:08:10,936 INFO [f8f85b0bb1d8:42949 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T16:08:10,936 INFO [f8f85b0bb1d8:42949 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-13T16:08:10,936 DEBUG [f8f85b0bb1d8:42949 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T16:08:10,943 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=f8f85b0bb1d8,43467,1731514088560 2024-11-13T16:08:10,950 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f8f85b0bb1d8,43467,1731514088560, state=OPENING 2024-11-13T16:08:10,961 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-13T16:08:10,970 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38187-0x1013621fa650002, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:08:10,970 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43467-0x1013621fa650001, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:08:10,970 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42949-0x1013621fa650000, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:08:10,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41333-0x1013621fa650003, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:08:10,972 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:08:10,972 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:08:10,973 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:08:10,973 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:08:10,976 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T16:08:10,979 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=f8f85b0bb1d8,43467,1731514088560}] 2024-11-13T16:08:11,156 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T16:08:11,159 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38923, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T16:08:11,173 INFO [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-13T16:08:11,174 INFO [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T16:08:11,175 INFO [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-13T16:08:11,179 INFO [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f8f85b0bb1d8%2C43467%2C1731514088560.meta, suffix=.meta, logDir=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/WALs/f8f85b0bb1d8,43467,1731514088560, archiveDir=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/oldWALs, maxLogs=32 2024-11-13T16:08:11,198 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/WALs/f8f85b0bb1d8,43467,1731514088560/f8f85b0bb1d8%2C43467%2C1731514088560.meta.1731514091182.meta, exclude list is [], retry=0 2024-11-13T16:08:11,203 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45743,DS-b848e0f5-c5b4-4830-9d9c-fc9502100de9,DISK] 2024-11-13T16:08:11,203 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33087,DS-83f68ee5-197b-4527-968f-c0dc23954597,DISK] 2024-11-13T16:08:11,204 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32863,DS-af4262a8-b437-4e57-821f-0315c01636b9,DISK] 2024-11-13T16:08:11,211 INFO [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/WALs/f8f85b0bb1d8,43467,1731514088560/f8f85b0bb1d8%2C43467%2C1731514088560.meta.1731514091182.meta 2024-11-13T16:08:11,211 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46455:46455),(127.0.0.1/127.0.0.1:38919:38919),(127.0.0.1/127.0.0.1:33713:33713)] 2024-11-13T16:08:11,212 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-13T16:08:11,213 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-13T16:08:11,216 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-13T16:08:11,223 INFO [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-13T16:08:11,228 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-13T16:08:11,229 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:11,230 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-13T16:08:11,230 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-13T16:08:11,234 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T16:08:11,236 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T16:08:11,237 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:11,238 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T16:08:11,238 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T16:08:11,240 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T16:08:11,240 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:11,241 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T16:08:11,242 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T16:08:11,244 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T16:08:11,244 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:11,245 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T16:08:11,246 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T16:08:11,249 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T16:08:11,249 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:11,250 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T16:08:11,252 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T16:08:11,254 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/hbase/meta/1588230740 2024-11-13T16:08:11,258 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/hbase/meta/1588230740 2024-11-13T16:08:11,267 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T16:08:11,268 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T16:08:11,269 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T16:08:11,274 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T16:08:11,277 INFO [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59360946, jitterRate=-0.11545297503471375}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T16:08:11,277 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-13T16:08:11,279 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731514091230Writing region info on filesystem at 1731514091231 (+1 ms)Initializing all the Stores at 1731514091233 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731514091233Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731514091234 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514091234Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731514091234Cleaning up temporary data from old regions at 1731514091268 (+34 ms)Running coprocessor post-open hooks at 1731514091277 (+9 ms)Region opened successfully at 1731514091279 (+2 ms) 2024-11-13T16:08:11,288 INFO [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731514091148 2024-11-13T16:08:11,305 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-13T16:08:11,306 INFO [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-13T16:08:11,309 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=f8f85b0bb1d8,43467,1731514088560 2024-11-13T16:08:11,312 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f8f85b0bb1d8,43467,1731514088560, state=OPEN 2024-11-13T16:08:11,325 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41333-0x1013621fa650003, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T16:08:11,326 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43467-0x1013621fa650001, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T16:08:11,326 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42949-0x1013621fa650000, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T16:08:11,326 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:08:11,326 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:08:11,326 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:08:11,326 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38187-0x1013621fa650002, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T16:08:11,326 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:08:11,327 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=f8f85b0bb1d8,43467,1731514088560 2024-11-13T16:08:11,343 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-13T16:08:11,343 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f8f85b0bb1d8,43467,1731514088560 in 352 msec 2024-11-13T16:08:11,351 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-13T16:08:11,351 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 846 msec 2024-11-13T16:08:11,353 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T16:08:11,353 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-13T16:08:11,371 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T16:08:11,373 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1] 2024-11-13T16:08:11,394 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T16:08:11,397 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36807, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T16:08:11,429 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2850 sec 2024-11-13T16:08:11,429 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731514091429, completionTime=-1 2024-11-13T16:08:11,432 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-13T16:08:11,433 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-13T16:08:11,511 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-13T16:08:11,511 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731514151511 2024-11-13T16:08:11,511 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731514211511 2024-11-13T16:08:11,511 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 78 msec 2024-11-13T16:08:11,514 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-13T16:08:11,537 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f8f85b0bb1d8,42949,1731514087490-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T16:08:11,538 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f8f85b0bb1d8,42949,1731514087490-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T16:08:11,538 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f8f85b0bb1d8,42949,1731514087490-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T16:08:11,540 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-f8f85b0bb1d8:42949, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T16:08:11,543 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-13T16:08:11,544 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-13T16:08:11,555 DEBUG [master/f8f85b0bb1d8:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-13T16:08:11,581 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.651sec 2024-11-13T16:08:11,583 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-13T16:08:11,585 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-13T16:08:11,586 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-13T16:08:11,586 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-13T16:08:11,586 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-13T16:08:11,587 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f8f85b0bb1d8,42949,1731514087490-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T16:08:11,588 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f8f85b0bb1d8,42949,1731514087490-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-13T16:08:11,593 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-13T16:08:11,594 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-13T16:08:11,594 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f8f85b0bb1d8,42949,1731514087490-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T16:08:11,688 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@359a87a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T16:08:11,692 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-13T16:08:11,692 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-13T16:08:11,695 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f8f85b0bb1d8,42949,-1 for getting cluster id 2024-11-13T16:08:11,698 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T16:08:11,708 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5d0c4b15-0fa1-4b67-8c04-53b67980164f' 2024-11-13T16:08:11,711 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T16:08:11,711 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5d0c4b15-0fa1-4b67-8c04-53b67980164f" 2024-11-13T16:08:11,714 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30eac8bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T16:08:11,714 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f8f85b0bb1d8,42949,-1] 2024-11-13T16:08:11,716 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T16:08:11,718 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T16:08:11,719 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45546, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T16:08:11,723 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3716250, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T16:08:11,723 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T16:08:11,731 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1] 2024-11-13T16:08:11,731 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T16:08:11,734 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58214, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T16:08:11,756 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=f8f85b0bb1d8,42949,1731514087490 2024-11-13T16:08:11,757 WARN [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false 2024-11-13T16:08:11,761 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T16:08:11,766 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is f8f85b0bb1d8,42949,1731514087490 2024-11-13T16:08:11,769 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7848b6a2 2024-11-13T16:08:11,771 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T16:08:11,774 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45556, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T16:08:11,783 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42949 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T16:08:11,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42949 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-13T16:08:11,811 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T16:08:11,814 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42949 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true" procId is: 4 2024-11-13T16:08:11,814 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:11,820 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T16:08:11,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T16:08:11,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741837_1013 (size=1020) 2024-11-13T16:08:11,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741837_1013 (size=1020) 2024-11-13T16:08:11,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741837_1013 (size=1020) 2024-11-13T16:08:11,857 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4aaf5667f132c289386d1b3b0c0ac57f, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731514091776.4aaf5667f132c289386d1b3b0c0ac57f.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46 2024-11-13T16:08:11,859 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 93f8ed38b68bd013eb0cf0e7615013ae, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731514091776.93f8ed38b68bd013eb0cf0e7615013ae.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46 2024-11-13T16:08:11,862 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 8ba796d80fc3b65f0948c95ccbdad39d, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731514091776.8ba796d80fc3b65f0948c95ccbdad39d.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46 2024-11-13T16:08:11,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741838_1014 (size=107) 2024-11-13T16:08:11,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741838_1014 (size=107) 2024-11-13T16:08:11,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741838_1014 (size=107) 2024-11-13T16:08:11,905 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731514091776.4aaf5667f132c289386d1b3b0c0ac57f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:11,905 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 4aaf5667f132c289386d1b3b0c0ac57f, disabling compactions & flushes 2024-11-13T16:08:11,905 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731514091776.4aaf5667f132c289386d1b3b0c0ac57f. 2024-11-13T16:08:11,905 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731514091776.4aaf5667f132c289386d1b3b0c0ac57f. 2024-11-13T16:08:11,906 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731514091776.4aaf5667f132c289386d1b3b0c0ac57f. after waiting 0 ms 2024-11-13T16:08:11,906 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731514091776.4aaf5667f132c289386d1b3b0c0ac57f. 2024-11-13T16:08:11,906 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731514091776.4aaf5667f132c289386d1b3b0c0ac57f. 2024-11-13T16:08:11,906 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4aaf5667f132c289386d1b3b0c0ac57f: Waiting for close lock at 1731514091905Disabling compacts and flushes for region at 1731514091905Disabling writes for close at 1731514091906 (+1 ms)Writing region close event to WAL at 1731514091906Closed at 1731514091906 2024-11-13T16:08:11,907 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e72e57b96ff9bc0bbd862ce564d79c42, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731514091776.e72e57b96ff9bc0bbd862ce564d79c42.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46 2024-11-13T16:08:11,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741839_1015 (size=110) 2024-11-13T16:08:11,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741839_1015 (size=110) 2024-11-13T16:08:11,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741839_1015 (size=110) 2024-11-13T16:08:11,923 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731514091776.93f8ed38b68bd013eb0cf0e7615013ae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:11,923 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing 93f8ed38b68bd013eb0cf0e7615013ae, disabling compactions & flushes 2024-11-13T16:08:11,923 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731514091776.93f8ed38b68bd013eb0cf0e7615013ae. 2024-11-13T16:08:11,923 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731514091776.93f8ed38b68bd013eb0cf0e7615013ae. 2024-11-13T16:08:11,923 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731514091776.93f8ed38b68bd013eb0cf0e7615013ae. after waiting 0 ms 2024-11-13T16:08:11,923 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731514091776.93f8ed38b68bd013eb0cf0e7615013ae. 2024-11-13T16:08:11,923 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731514091776.93f8ed38b68bd013eb0cf0e7615013ae. 2024-11-13T16:08:11,923 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for 93f8ed38b68bd013eb0cf0e7615013ae: Waiting for close lock at 1731514091923Disabling compacts and flushes for region at 1731514091923Disabling writes for close at 1731514091923Writing region close event to WAL at 1731514091923Closed at 1731514091923 2024-11-13T16:08:11,924 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 34f3d3b6a6dae1f4077567bed0bd8e75, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731514091776.34f3d3b6a6dae1f4077567bed0bd8e75.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46 2024-11-13T16:08:11,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741840_1016 (size=110) 2024-11-13T16:08:11,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741840_1016 (size=110) 2024-11-13T16:08:11,927 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731514091776.8ba796d80fc3b65f0948c95ccbdad39d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:11,927 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing 8ba796d80fc3b65f0948c95ccbdad39d, disabling compactions & flushes 2024-11-13T16:08:11,928 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731514091776.8ba796d80fc3b65f0948c95ccbdad39d. 2024-11-13T16:08:11,928 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731514091776.8ba796d80fc3b65f0948c95ccbdad39d. 2024-11-13T16:08:11,928 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731514091776.8ba796d80fc3b65f0948c95ccbdad39d. after waiting 0 ms 2024-11-13T16:08:11,928 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731514091776.8ba796d80fc3b65f0948c95ccbdad39d. 2024-11-13T16:08:11,928 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731514091776.8ba796d80fc3b65f0948c95ccbdad39d. 2024-11-13T16:08:11,928 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for 8ba796d80fc3b65f0948c95ccbdad39d: Waiting for close lock at 1731514091927Disabling compacts and flushes for region at 1731514091927Disabling writes for close at 1731514091928 (+1 ms)Writing region close event to WAL at 1731514091928Closed at 1731514091928 2024-11-13T16:08:11,929 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 3945ce860c8642051e17517944a83136, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731514091776.3945ce860c8642051e17517944a83136.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46 2024-11-13T16:08:11,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741840_1016 (size=110) 2024-11-13T16:08:11,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741841_1017 (size=110) 2024-11-13T16:08:11,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741841_1017 (size=110) 2024-11-13T16:08:11,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T16:08:11,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741841_1017 (size=110) 2024-11-13T16:08:11,942 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731514091776.e72e57b96ff9bc0bbd862ce564d79c42.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:11,942 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing e72e57b96ff9bc0bbd862ce564d79c42, disabling compactions & flushes 2024-11-13T16:08:11,942 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731514091776.e72e57b96ff9bc0bbd862ce564d79c42. 2024-11-13T16:08:11,942 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731514091776.e72e57b96ff9bc0bbd862ce564d79c42. 2024-11-13T16:08:11,942 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731514091776.e72e57b96ff9bc0bbd862ce564d79c42. after waiting 0 ms 2024-11-13T16:08:11,943 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731514091776.e72e57b96ff9bc0bbd862ce564d79c42. 2024-11-13T16:08:11,943 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731514091776.e72e57b96ff9bc0bbd862ce564d79c42. 2024-11-13T16:08:11,943 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for e72e57b96ff9bc0bbd862ce564d79c42: Waiting for close lock at 1731514091942Disabling compacts and flushes for region at 1731514091942Disabling writes for close at 1731514091942Writing region close event to WAL at 1731514091943 (+1 ms)Closed at 1731514091943 2024-11-13T16:08:11,944 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1fcf78578ccab75f1705f3c2d7bcdb14, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731514091776.1fcf78578ccab75f1705f3c2d7bcdb14.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46 2024-11-13T16:08:11,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741842_1018 (size=110) 2024-11-13T16:08:11,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741842_1018 (size=110) 2024-11-13T16:08:11,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741842_1018 (size=110) 2024-11-13T16:08:11,966 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731514091776.34f3d3b6a6dae1f4077567bed0bd8e75.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:11,966 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing 34f3d3b6a6dae1f4077567bed0bd8e75, disabling compactions & flushes 2024-11-13T16:08:11,966 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731514091776.34f3d3b6a6dae1f4077567bed0bd8e75. 2024-11-13T16:08:11,967 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731514091776.34f3d3b6a6dae1f4077567bed0bd8e75. 2024-11-13T16:08:11,967 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731514091776.34f3d3b6a6dae1f4077567bed0bd8e75. after waiting 0 ms 2024-11-13T16:08:11,967 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731514091776.34f3d3b6a6dae1f4077567bed0bd8e75. 2024-11-13T16:08:11,967 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731514091776.34f3d3b6a6dae1f4077567bed0bd8e75. 2024-11-13T16:08:11,967 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for 34f3d3b6a6dae1f4077567bed0bd8e75: Waiting for close lock at 1731514091966Disabling compacts and flushes for region at 1731514091966Disabling writes for close at 1731514091967 (+1 ms)Writing region close event to WAL at 1731514091967Closed at 1731514091967 2024-11-13T16:08:11,968 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 1a610c2403a0b491d368473a6e1fb049, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731514091776.1a610c2403a0b491d368473a6e1fb049.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46 2024-11-13T16:08:11,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741843_1019 (size=110) 2024-11-13T16:08:11,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741843_1019 (size=110) 2024-11-13T16:08:11,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741843_1019 (size=110) 2024-11-13T16:08:11,976 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731514091776.3945ce860c8642051e17517944a83136.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:11,977 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing 3945ce860c8642051e17517944a83136, disabling compactions & flushes 2024-11-13T16:08:11,977 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731514091776.3945ce860c8642051e17517944a83136. 2024-11-13T16:08:11,977 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731514091776.3945ce860c8642051e17517944a83136. 2024-11-13T16:08:11,977 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731514091776.3945ce860c8642051e17517944a83136. after waiting 0 ms 2024-11-13T16:08:11,977 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731514091776.3945ce860c8642051e17517944a83136. 2024-11-13T16:08:11,977 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731514091776.3945ce860c8642051e17517944a83136. 2024-11-13T16:08:11,977 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for 3945ce860c8642051e17517944a83136: Waiting for close lock at 1731514091976Disabling compacts and flushes for region at 1731514091976Disabling writes for close at 1731514091977 (+1 ms)Writing region close event to WAL at 1731514091977Closed at 1731514091977 2024-11-13T16:08:11,978 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => e8725a794e6d251e1bf6e3502baa7700, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731514091776.e8725a794e6d251e1bf6e3502baa7700.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46 2024-11-13T16:08:11,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741844_1020 (size=110) 2024-11-13T16:08:11,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741844_1020 (size=110) 2024-11-13T16:08:11,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741844_1020 (size=110) 2024-11-13T16:08:11,994 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731514091776.1fcf78578ccab75f1705f3c2d7bcdb14.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:11,995 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 1fcf78578ccab75f1705f3c2d7bcdb14, disabling compactions & flushes 2024-11-13T16:08:11,995 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731514091776.1fcf78578ccab75f1705f3c2d7bcdb14. 2024-11-13T16:08:11,995 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731514091776.1fcf78578ccab75f1705f3c2d7bcdb14. 2024-11-13T16:08:11,995 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731514091776.1fcf78578ccab75f1705f3c2d7bcdb14. after waiting 0 ms 2024-11-13T16:08:11,995 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731514091776.1fcf78578ccab75f1705f3c2d7bcdb14. 2024-11-13T16:08:11,995 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731514091776.1fcf78578ccab75f1705f3c2d7bcdb14. 2024-11-13T16:08:11,995 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1fcf78578ccab75f1705f3c2d7bcdb14: Waiting for close lock at 1731514091995Disabling compacts and flushes for region at 1731514091995Disabling writes for close at 1731514091995Writing region close event to WAL at 1731514091995Closed at 1731514091995 2024-11-13T16:08:11,996 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fa6636be3559bc84793534420cd4f207, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731514091776.fa6636be3559bc84793534420cd4f207.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46 2024-11-13T16:08:12,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741845_1021 (size=110) 2024-11-13T16:08:12,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741845_1021 (size=110) 2024-11-13T16:08:12,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741845_1021 (size=110) 2024-11-13T16:08:12,019 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731514091776.1a610c2403a0b491d368473a6e1fb049.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:12,019 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing 1a610c2403a0b491d368473a6e1fb049, disabling compactions & flushes 2024-11-13T16:08:12,019 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731514091776.1a610c2403a0b491d368473a6e1fb049. 2024-11-13T16:08:12,019 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731514091776.1a610c2403a0b491d368473a6e1fb049. 2024-11-13T16:08:12,019 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731514091776.1a610c2403a0b491d368473a6e1fb049. after waiting 0 ms 2024-11-13T16:08:12,019 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731514091776.1a610c2403a0b491d368473a6e1fb049. 2024-11-13T16:08:12,020 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731514091776.1a610c2403a0b491d368473a6e1fb049. 2024-11-13T16:08:12,020 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for 1a610c2403a0b491d368473a6e1fb049: Waiting for close lock at 1731514092019Disabling compacts and flushes for region at 1731514092019Disabling writes for close at 1731514092019Writing region close event to WAL at 1731514092020 (+1 ms)Closed at 1731514092020 2024-11-13T16:08:12,021 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 05d65f4a6bfecb3d530d15832c218252, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731514091776.05d65f4a6bfecb3d530d15832c218252.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46 2024-11-13T16:08:12,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741846_1022 (size=110) 2024-11-13T16:08:12,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741846_1022 (size=110) 2024-11-13T16:08:12,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741846_1022 (size=110) 2024-11-13T16:08:12,027 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731514091776.e8725a794e6d251e1bf6e3502baa7700.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:12,027 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing e8725a794e6d251e1bf6e3502baa7700, disabling compactions & flushes 2024-11-13T16:08:12,027 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731514091776.e8725a794e6d251e1bf6e3502baa7700. 2024-11-13T16:08:12,027 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731514091776.e8725a794e6d251e1bf6e3502baa7700. 2024-11-13T16:08:12,028 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731514091776.e8725a794e6d251e1bf6e3502baa7700. after waiting 0 ms 2024-11-13T16:08:12,028 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731514091776.e8725a794e6d251e1bf6e3502baa7700. 2024-11-13T16:08:12,028 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731514091776.e8725a794e6d251e1bf6e3502baa7700. 2024-11-13T16:08:12,028 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for e8725a794e6d251e1bf6e3502baa7700: Waiting for close lock at 1731514092027Disabling compacts and flushes for region at 1731514092027Disabling writes for close at 1731514092028 (+1 ms)Writing region close event to WAL at 1731514092028Closed at 1731514092028 2024-11-13T16:08:12,028 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 78285ca993cc7fbfd78041c30558f756, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731514091776.78285ca993cc7fbfd78041c30558f756.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46 2024-11-13T16:08:12,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741847_1023 (size=110) 2024-11-13T16:08:12,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741847_1023 (size=110) 2024-11-13T16:08:12,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741847_1023 (size=110) 2024-11-13T16:08:12,036 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731514091776.fa6636be3559bc84793534420cd4f207.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:12,036 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing fa6636be3559bc84793534420cd4f207, disabling compactions & flushes 2024-11-13T16:08:12,036 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731514091776.fa6636be3559bc84793534420cd4f207. 2024-11-13T16:08:12,036 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731514091776.fa6636be3559bc84793534420cd4f207. 2024-11-13T16:08:12,036 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731514091776.fa6636be3559bc84793534420cd4f207. after waiting 0 ms 2024-11-13T16:08:12,036 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731514091776.fa6636be3559bc84793534420cd4f207. 2024-11-13T16:08:12,036 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731514091776.fa6636be3559bc84793534420cd4f207. 2024-11-13T16:08:12,036 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for fa6636be3559bc84793534420cd4f207: Waiting for close lock at 1731514092036Disabling compacts and flushes for region at 1731514092036Disabling writes for close at 1731514092036Writing region close event to WAL at 1731514092036Closed at 1731514092036 2024-11-13T16:08:12,037 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => aa124b802c708ca1f715cb1f8c9c4eaa, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731514091776.aa124b802c708ca1f715cb1f8c9c4eaa.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46 2024-11-13T16:08:12,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741849_1025 (size=110) 2024-11-13T16:08:12,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741849_1025 (size=110) 2024-11-13T16:08:12,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741849_1025 (size=110) 2024-11-13T16:08:12,067 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731514091776.78285ca993cc7fbfd78041c30558f756.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:12,067 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing 78285ca993cc7fbfd78041c30558f756, disabling compactions & flushes 2024-11-13T16:08:12,067 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731514091776.78285ca993cc7fbfd78041c30558f756. 2024-11-13T16:08:12,067 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731514091776.78285ca993cc7fbfd78041c30558f756. 2024-11-13T16:08:12,068 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731514091776.78285ca993cc7fbfd78041c30558f756. after waiting 0 ms 2024-11-13T16:08:12,068 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731514091776.78285ca993cc7fbfd78041c30558f756. 2024-11-13T16:08:12,068 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731514091776.78285ca993cc7fbfd78041c30558f756. 2024-11-13T16:08:12,068 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for 78285ca993cc7fbfd78041c30558f756: Waiting for close lock at 1731514092067Disabling compacts and flushes for region at 1731514092067Disabling writes for close at 1731514092068 (+1 ms)Writing region close event to WAL at 1731514092068Closed at 1731514092068 2024-11-13T16:08:12,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741848_1024 (size=110) 2024-11-13T16:08:12,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741848_1024 (size=110) 2024-11-13T16:08:12,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741850_1026 (size=110) 2024-11-13T16:08:12,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741848_1024 (size=110) 2024-11-13T16:08:12,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741850_1026 (size=110) 2024-11-13T16:08:12,069 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 3647edbd8aef23dda3887b61277da1d3, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731514091776.3647edbd8aef23dda3887b61277da1d3.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46 2024-11-13T16:08:12,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741850_1026 (size=110) 2024-11-13T16:08:12,070 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731514091776.aa124b802c708ca1f715cb1f8c9c4eaa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:12,070 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing aa124b802c708ca1f715cb1f8c9c4eaa, disabling compactions & flushes 2024-11-13T16:08:12,070 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731514091776.aa124b802c708ca1f715cb1f8c9c4eaa. 2024-11-13T16:08:12,070 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731514091776.aa124b802c708ca1f715cb1f8c9c4eaa. 2024-11-13T16:08:12,070 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731514091776.aa124b802c708ca1f715cb1f8c9c4eaa. after waiting 0 ms 2024-11-13T16:08:12,070 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731514091776.aa124b802c708ca1f715cb1f8c9c4eaa. 2024-11-13T16:08:12,070 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731514091776.aa124b802c708ca1f715cb1f8c9c4eaa. 2024-11-13T16:08:12,070 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for aa124b802c708ca1f715cb1f8c9c4eaa: Waiting for close lock at 1731514092070Disabling compacts and flushes for region at 1731514092070Disabling writes for close at 1731514092070Writing region close event to WAL at 1731514092070Closed at 1731514092070 2024-11-13T16:08:12,071 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 29df624bbcb878edce7fcdf0abceea88, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731514091776.29df624bbcb878edce7fcdf0abceea88.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46 2024-11-13T16:08:12,072 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731514091776.05d65f4a6bfecb3d530d15832c218252.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:12,072 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing 05d65f4a6bfecb3d530d15832c218252, disabling compactions & flushes 2024-11-13T16:08:12,072 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731514091776.05d65f4a6bfecb3d530d15832c218252. 2024-11-13T16:08:12,072 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731514091776.05d65f4a6bfecb3d530d15832c218252. 2024-11-13T16:08:12,072 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731514091776.05d65f4a6bfecb3d530d15832c218252. after waiting 0 ms 2024-11-13T16:08:12,072 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731514091776.05d65f4a6bfecb3d530d15832c218252. 2024-11-13T16:08:12,072 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731514091776.05d65f4a6bfecb3d530d15832c218252. 2024-11-13T16:08:12,072 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for 05d65f4a6bfecb3d530d15832c218252: Waiting for close lock at 1731514092072Disabling compacts and flushes for region at 1731514092072Disabling writes for close at 1731514092072Writing region close event to WAL at 1731514092072Closed at 1731514092072 2024-11-13T16:08:12,073 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => fe97fe0e9b37811ba99ca8100b461589, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731514091776.fe97fe0e9b37811ba99ca8100b461589.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46 2024-11-13T16:08:12,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741851_1027 (size=110) 2024-11-13T16:08:12,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741851_1027 (size=110) 2024-11-13T16:08:12,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741851_1027 (size=110) 2024-11-13T16:08:12,085 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731514091776.3647edbd8aef23dda3887b61277da1d3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:12,085 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing 3647edbd8aef23dda3887b61277da1d3, disabling compactions & flushes 2024-11-13T16:08:12,086 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731514091776.3647edbd8aef23dda3887b61277da1d3. 2024-11-13T16:08:12,086 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731514091776.3647edbd8aef23dda3887b61277da1d3. 2024-11-13T16:08:12,086 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731514091776.3647edbd8aef23dda3887b61277da1d3. after waiting 0 ms 2024-11-13T16:08:12,086 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731514091776.3647edbd8aef23dda3887b61277da1d3. 2024-11-13T16:08:12,086 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731514091776.3647edbd8aef23dda3887b61277da1d3. 2024-11-13T16:08:12,086 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for 3647edbd8aef23dda3887b61277da1d3: Waiting for close lock at 1731514092085Disabling compacts and flushes for region at 1731514092085Disabling writes for close at 1731514092086 (+1 ms)Writing region close event to WAL at 1731514092086Closed at 1731514092086 2024-11-13T16:08:12,087 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 7c34e9ded13254b91545186c85befbf8, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731514091776.7c34e9ded13254b91545186c85befbf8.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46 2024-11-13T16:08:12,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741852_1028 (size=110) 2024-11-13T16:08:12,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741852_1028 (size=110) 2024-11-13T16:08:12,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741852_1028 (size=110) 2024-11-13T16:08:12,090 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731514091776.29df624bbcb878edce7fcdf0abceea88.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:12,090 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 29df624bbcb878edce7fcdf0abceea88, disabling compactions & flushes 2024-11-13T16:08:12,090 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731514091776.29df624bbcb878edce7fcdf0abceea88. 2024-11-13T16:08:12,091 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731514091776.29df624bbcb878edce7fcdf0abceea88. 2024-11-13T16:08:12,091 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731514091776.29df624bbcb878edce7fcdf0abceea88. after waiting 0 ms 2024-11-13T16:08:12,091 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731514091776.29df624bbcb878edce7fcdf0abceea88. 2024-11-13T16:08:12,091 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731514091776.29df624bbcb878edce7fcdf0abceea88. 2024-11-13T16:08:12,091 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 29df624bbcb878edce7fcdf0abceea88: Waiting for close lock at 1731514092090Disabling compacts and flushes for region at 1731514092090Disabling writes for close at 1731514092091 (+1 ms)Writing region close event to WAL at 1731514092091Closed at 1731514092091 2024-11-13T16:08:12,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741853_1029 (size=110) 2024-11-13T16:08:12,092 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f8f8b8e4ae8470fd8a1d2bf0684947ef, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731514091776.f8f8b8e4ae8470fd8a1d2bf0684947ef.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46 2024-11-13T16:08:12,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741853_1029 (size=110) 2024-11-13T16:08:12,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741853_1029 (size=110) 2024-11-13T16:08:12,097 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731514091776.fe97fe0e9b37811ba99ca8100b461589.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:12,097 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing fe97fe0e9b37811ba99ca8100b461589, disabling compactions & flushes 2024-11-13T16:08:12,097 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731514091776.fe97fe0e9b37811ba99ca8100b461589. 2024-11-13T16:08:12,097 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731514091776.fe97fe0e9b37811ba99ca8100b461589. 2024-11-13T16:08:12,097 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731514091776.fe97fe0e9b37811ba99ca8100b461589. after waiting 0 ms 2024-11-13T16:08:12,097 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731514091776.fe97fe0e9b37811ba99ca8100b461589. 2024-11-13T16:08:12,097 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731514091776.fe97fe0e9b37811ba99ca8100b461589. 2024-11-13T16:08:12,097 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for fe97fe0e9b37811ba99ca8100b461589: Waiting for close lock at 1731514092097Disabling compacts and flushes for region at 1731514092097Disabling writes for close at 1731514092097Writing region close event to WAL at 1731514092097Closed at 1731514092097 2024-11-13T16:08:12,098 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => fec5f21b48c36bf01685ec2ee58f0490, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731514091776.fec5f21b48c36bf01685ec2ee58f0490.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46 2024-11-13T16:08:12,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741854_1030 (size=110) 2024-11-13T16:08:12,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741854_1030 (size=110) 2024-11-13T16:08:12,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741854_1030 (size=110) 2024-11-13T16:08:12,107 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731514091776.7c34e9ded13254b91545186c85befbf8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:12,107 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing 7c34e9ded13254b91545186c85befbf8, disabling compactions & flushes 2024-11-13T16:08:12,107 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731514091776.7c34e9ded13254b91545186c85befbf8. 2024-11-13T16:08:12,108 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731514091776.7c34e9ded13254b91545186c85befbf8. 2024-11-13T16:08:12,108 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731514091776.7c34e9ded13254b91545186c85befbf8. after waiting 0 ms 2024-11-13T16:08:12,108 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731514091776.7c34e9ded13254b91545186c85befbf8. 2024-11-13T16:08:12,108 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731514091776.7c34e9ded13254b91545186c85befbf8. 2024-11-13T16:08:12,108 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for 7c34e9ded13254b91545186c85befbf8: Waiting for close lock at 1731514092107Disabling compacts and flushes for region at 1731514092107Disabling writes for close at 1731514092108 (+1 ms)Writing region close event to WAL at 1731514092108Closed at 1731514092108 2024-11-13T16:08:12,109 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 97998e7179d85b23a3f5928cd6511629, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731514091776.97998e7179d85b23a3f5928cd6511629.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46 2024-11-13T16:08:12,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741855_1031 (size=110) 2024-11-13T16:08:12,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741856_1032 (size=110) 2024-11-13T16:08:12,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741855_1031 (size=110) 2024-11-13T16:08:12,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741856_1032 (size=110) 2024-11-13T16:08:12,115 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731514091776.fec5f21b48c36bf01685ec2ee58f0490.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:12,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741856_1032 (size=110) 2024-11-13T16:08:12,115 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing fec5f21b48c36bf01685ec2ee58f0490, disabling compactions & flushes 2024-11-13T16:08:12,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741855_1031 (size=110) 2024-11-13T16:08:12,115 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731514091776.fec5f21b48c36bf01685ec2ee58f0490. 2024-11-13T16:08:12,115 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731514091776.fec5f21b48c36bf01685ec2ee58f0490. 2024-11-13T16:08:12,115 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731514091776.fec5f21b48c36bf01685ec2ee58f0490. after waiting 0 ms 2024-11-13T16:08:12,115 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731514091776.fec5f21b48c36bf01685ec2ee58f0490. 2024-11-13T16:08:12,115 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731514091776.fec5f21b48c36bf01685ec2ee58f0490. 2024-11-13T16:08:12,115 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for fec5f21b48c36bf01685ec2ee58f0490: Waiting for close lock at 1731514092115Disabling compacts and flushes for region at 1731514092115Disabling writes for close at 1731514092115Writing region close event to WAL at 1731514092115Closed at 1731514092115 2024-11-13T16:08:12,116 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 519ece296a21b5ddccc38a7c928780bb, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731514091776.519ece296a21b5ddccc38a7c928780bb.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46 2024-11-13T16:08:12,119 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731514091776.f8f8b8e4ae8470fd8a1d2bf0684947ef.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:12,119 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing f8f8b8e4ae8470fd8a1d2bf0684947ef, disabling compactions & flushes 2024-11-13T16:08:12,119 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731514091776.f8f8b8e4ae8470fd8a1d2bf0684947ef. 2024-11-13T16:08:12,119 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731514091776.f8f8b8e4ae8470fd8a1d2bf0684947ef. 2024-11-13T16:08:12,119 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731514091776.f8f8b8e4ae8470fd8a1d2bf0684947ef. after waiting 0 ms 2024-11-13T16:08:12,119 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731514091776.f8f8b8e4ae8470fd8a1d2bf0684947ef. 2024-11-13T16:08:12,119 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731514091776.f8f8b8e4ae8470fd8a1d2bf0684947ef. 2024-11-13T16:08:12,119 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for f8f8b8e4ae8470fd8a1d2bf0684947ef: Waiting for close lock at 1731514092119Disabling compacts and flushes for region at 1731514092119Disabling writes for close at 1731514092119Writing region close event to WAL at 1731514092119Closed at 1731514092119 2024-11-13T16:08:12,120 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c4a87adf770b37d40ab6691e377ad1b5, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731514091776.c4a87adf770b37d40ab6691e377ad1b5.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46 2024-11-13T16:08:12,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741857_1033 (size=110) 2024-11-13T16:08:12,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741857_1033 (size=110) 2024-11-13T16:08:12,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741857_1033 (size=110) 2024-11-13T16:08:12,135 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731514091776.97998e7179d85b23a3f5928cd6511629.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:12,135 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing 97998e7179d85b23a3f5928cd6511629, disabling compactions & flushes 2024-11-13T16:08:12,135 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731514091776.97998e7179d85b23a3f5928cd6511629. 2024-11-13T16:08:12,135 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731514091776.97998e7179d85b23a3f5928cd6511629. 2024-11-13T16:08:12,135 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731514091776.97998e7179d85b23a3f5928cd6511629. after waiting 0 ms 2024-11-13T16:08:12,135 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731514091776.97998e7179d85b23a3f5928cd6511629. 2024-11-13T16:08:12,135 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731514091776.97998e7179d85b23a3f5928cd6511629. 2024-11-13T16:08:12,135 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for 97998e7179d85b23a3f5928cd6511629: Waiting for close lock at 1731514092135Disabling compacts and flushes for region at 1731514092135Disabling writes for close at 1731514092135Writing region close event to WAL at 1731514092135Closed at 1731514092135 2024-11-13T16:08:12,136 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => bf1f9f9bc1bc9ffce566d41f8ae3619a, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731514091776.bf1f9f9bc1bc9ffce566d41f8ae3619a.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46 2024-11-13T16:08:12,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741859_1035 (size=110) 2024-11-13T16:08:12,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741859_1035 (size=110) 2024-11-13T16:08:12,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741859_1035 (size=110) 2024-11-13T16:08:12,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741858_1034 (size=110) 2024-11-13T16:08:12,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741858_1034 (size=110) 2024-11-13T16:08:12,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T16:08:12,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741858_1034 (size=110) 2024-11-13T16:08:12,149 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731514091776.519ece296a21b5ddccc38a7c928780bb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:12,150 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing 519ece296a21b5ddccc38a7c928780bb, disabling compactions & flushes 2024-11-13T16:08:12,150 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731514091776.519ece296a21b5ddccc38a7c928780bb. 2024-11-13T16:08:12,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741860_1036 (size=110) 2024-11-13T16:08:12,150 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731514091776.519ece296a21b5ddccc38a7c928780bb. 2024-11-13T16:08:12,150 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731514091776.519ece296a21b5ddccc38a7c928780bb. after waiting 0 ms 2024-11-13T16:08:12,150 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731514091776.519ece296a21b5ddccc38a7c928780bb. 2024-11-13T16:08:12,150 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731514091776.519ece296a21b5ddccc38a7c928780bb. 2024-11-13T16:08:12,150 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for 519ece296a21b5ddccc38a7c928780bb: Waiting for close lock at 1731514092150Disabling compacts and flushes for region at 1731514092150Disabling writes for close at 1731514092150Writing region close event to WAL at 1731514092150Closed at 1731514092150 2024-11-13T16:08:12,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741860_1036 (size=110) 2024-11-13T16:08:12,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741860_1036 (size=110) 2024-11-13T16:08:12,151 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 4a146f98682f25e8cddcae5dbf318603, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731514091776.4a146f98682f25e8cddcae5dbf318603.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46 2024-11-13T16:08:12,151 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731514091776.bf1f9f9bc1bc9ffce566d41f8ae3619a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:12,152 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing bf1f9f9bc1bc9ffce566d41f8ae3619a, disabling compactions & flushes 2024-11-13T16:08:12,152 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731514091776.bf1f9f9bc1bc9ffce566d41f8ae3619a. 2024-11-13T16:08:12,152 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731514091776.bf1f9f9bc1bc9ffce566d41f8ae3619a. 2024-11-13T16:08:12,152 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731514091776.bf1f9f9bc1bc9ffce566d41f8ae3619a. after waiting 0 ms 2024-11-13T16:08:12,152 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731514091776.bf1f9f9bc1bc9ffce566d41f8ae3619a. 2024-11-13T16:08:12,152 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731514091776.bf1f9f9bc1bc9ffce566d41f8ae3619a. 2024-11-13T16:08:12,152 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for bf1f9f9bc1bc9ffce566d41f8ae3619a: Waiting for close lock at 1731514092152Disabling compacts and flushes for region at 1731514092152Disabling writes for close at 1731514092152Writing region close event to WAL at 1731514092152Closed at 1731514092152 2024-11-13T16:08:12,153 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 6be98d350311c17b11fdb9e9a1238fca, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731514091776.6be98d350311c17b11fdb9e9a1238fca.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46 2024-11-13T16:08:12,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741862_1038 (size=110) 2024-11-13T16:08:12,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741862_1038 (size=110) 2024-11-13T16:08:12,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741862_1038 (size=110) 2024-11-13T16:08:12,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741861_1037 (size=110) 2024-11-13T16:08:12,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741861_1037 (size=110) 2024-11-13T16:08:12,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741861_1037 (size=110) 2024-11-13T16:08:12,172 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731514091776.4a146f98682f25e8cddcae5dbf318603.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:12,172 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing 4a146f98682f25e8cddcae5dbf318603, disabling compactions & flushes 2024-11-13T16:08:12,173 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731514091776.4a146f98682f25e8cddcae5dbf318603. 2024-11-13T16:08:12,173 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731514091776.4a146f98682f25e8cddcae5dbf318603. 2024-11-13T16:08:12,173 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731514091776.4a146f98682f25e8cddcae5dbf318603. after waiting 0 ms 2024-11-13T16:08:12,173 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731514091776.4a146f98682f25e8cddcae5dbf318603. 2024-11-13T16:08:12,173 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731514091776.4a146f98682f25e8cddcae5dbf318603. 2024-11-13T16:08:12,173 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for 4a146f98682f25e8cddcae5dbf318603: Waiting for close lock at 1731514092172Disabling compacts and flushes for region at 1731514092172Disabling writes for close at 1731514092173 (+1 ms)Writing region close event to WAL at 1731514092173Closed at 1731514092173 2024-11-13T16:08:12,173 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 72d273770314688e78624993de4f3398, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731514091776.72d273770314688e78624993de4f3398.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46 2024-11-13T16:08:12,175 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731514091776.6be98d350311c17b11fdb9e9a1238fca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:12,175 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing 6be98d350311c17b11fdb9e9a1238fca, disabling compactions & flushes 2024-11-13T16:08:12,175 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731514091776.6be98d350311c17b11fdb9e9a1238fca. 2024-11-13T16:08:12,175 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731514091776.6be98d350311c17b11fdb9e9a1238fca. 2024-11-13T16:08:12,175 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731514091776.6be98d350311c17b11fdb9e9a1238fca. after waiting 0 ms 2024-11-13T16:08:12,175 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731514091776.6be98d350311c17b11fdb9e9a1238fca. 2024-11-13T16:08:12,175 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731514091776.6be98d350311c17b11fdb9e9a1238fca. 2024-11-13T16:08:12,176 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for 6be98d350311c17b11fdb9e9a1238fca: Waiting for close lock at 1731514092175Disabling compacts and flushes for region at 1731514092175Disabling writes for close at 1731514092175Writing region close event to WAL at 1731514092175Closed at 1731514092175 2024-11-13T16:08:12,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741863_1039 (size=107) 2024-11-13T16:08:12,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741863_1039 (size=107) 2024-11-13T16:08:12,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741863_1039 (size=107) 2024-11-13T16:08:12,189 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731514091776.72d273770314688e78624993de4f3398.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:12,189 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing 72d273770314688e78624993de4f3398, disabling compactions & flushes 2024-11-13T16:08:12,189 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731514091776.72d273770314688e78624993de4f3398. 2024-11-13T16:08:12,189 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731514091776.72d273770314688e78624993de4f3398. 2024-11-13T16:08:12,189 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731514091776.72d273770314688e78624993de4f3398. after waiting 0 ms 2024-11-13T16:08:12,189 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731514091776.72d273770314688e78624993de4f3398. 2024-11-13T16:08:12,189 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731514091776.72d273770314688e78624993de4f3398. 2024-11-13T16:08:12,190 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for 72d273770314688e78624993de4f3398: Waiting for close lock at 1731514092189Disabling compacts and flushes for region at 1731514092189Disabling writes for close at 1731514092189Writing region close event to WAL at 1731514092189Closed at 1731514092189 2024-11-13T16:08:12,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T16:08:12,545 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731514091776.c4a87adf770b37d40ab6691e377ad1b5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:12,545 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing c4a87adf770b37d40ab6691e377ad1b5, disabling compactions & flushes 2024-11-13T16:08:12,545 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731514091776.c4a87adf770b37d40ab6691e377ad1b5. 2024-11-13T16:08:12,545 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731514091776.c4a87adf770b37d40ab6691e377ad1b5. 2024-11-13T16:08:12,545 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731514091776.c4a87adf770b37d40ab6691e377ad1b5. after waiting 0 ms 2024-11-13T16:08:12,545 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731514091776.c4a87adf770b37d40ab6691e377ad1b5. 2024-11-13T16:08:12,546 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731514091776.c4a87adf770b37d40ab6691e377ad1b5. 2024-11-13T16:08:12,546 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for c4a87adf770b37d40ab6691e377ad1b5: Waiting for close lock at 1731514092545Disabling compacts and flushes for region at 1731514092545Disabling writes for close at 1731514092545Writing region close event to WAL at 1731514092545Closed at 1731514092545 2024-11-13T16:08:12,549 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T16:08:12,556 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731514091776.4aaf5667f132c289386d1b3b0c0ac57f.","families":{"info":[{"qualifier":"regioninfo","vlen":106,"tag":[],"timestamp":"1731514092549"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731514092549"}]},"ts":"1731514092549"} 2024-11-13T16:08:12,556 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731514091776.93f8ed38b68bd013eb0cf0e7615013ae.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731514092549"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731514092549"}]},"ts":"1731514092549"} 2024-11-13T16:08:12,556 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731514091776.8ba796d80fc3b65f0948c95ccbdad39d.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731514092549"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731514092549"}]},"ts":"1731514092549"} 2024-11-13T16:08:12,557 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731514091776.e72e57b96ff9bc0bbd862ce564d79c42.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731514092549"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731514092549"}]},"ts":"1731514092549"} 2024-11-13T16:08:12,557 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731514091776.34f3d3b6a6dae1f4077567bed0bd8e75.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731514092549"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731514092549"}]},"ts":"1731514092549"} 2024-11-13T16:08:12,557 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731514091776.3945ce860c8642051e17517944a83136.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731514092549"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731514092549"}]},"ts":"1731514092549"} 2024-11-13T16:08:12,557 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731514091776.1fcf78578ccab75f1705f3c2d7bcdb14.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731514092549"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731514092549"}]},"ts":"1731514092549"} 2024-11-13T16:08:12,557 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731514091776.1a610c2403a0b491d368473a6e1fb049.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731514092549"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731514092549"}]},"ts":"1731514092549"} 2024-11-13T16:08:12,557 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731514091776.e8725a794e6d251e1bf6e3502baa7700.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731514092549"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731514092549"}]},"ts":"1731514092549"} 2024-11-13T16:08:12,558 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731514091776.fa6636be3559bc84793534420cd4f207.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731514092549"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731514092549"}]},"ts":"1731514092549"} 2024-11-13T16:08:12,558 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731514091776.78285ca993cc7fbfd78041c30558f756.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731514092549"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731514092549"}]},"ts":"1731514092549"} 2024-11-13T16:08:12,558 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731514091776.aa124b802c708ca1f715cb1f8c9c4eaa.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731514092549"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731514092549"}]},"ts":"1731514092549"} 2024-11-13T16:08:12,558 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731514091776.05d65f4a6bfecb3d530d15832c218252.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731514092549"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731514092549"}]},"ts":"1731514092549"} 2024-11-13T16:08:12,558 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731514091776.3647edbd8aef23dda3887b61277da1d3.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731514092549"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731514092549"}]},"ts":"1731514092549"} 2024-11-13T16:08:12,559 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731514091776.29df624bbcb878edce7fcdf0abceea88.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731514092549"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731514092549"}]},"ts":"1731514092549"} 2024-11-13T16:08:12,559 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731514091776.fe97fe0e9b37811ba99ca8100b461589.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731514092549"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731514092549"}]},"ts":"1731514092549"} 2024-11-13T16:08:12,559 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731514091776.7c34e9ded13254b91545186c85befbf8.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731514092549"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731514092549"}]},"ts":"1731514092549"} 2024-11-13T16:08:12,559 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731514091776.fec5f21b48c36bf01685ec2ee58f0490.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731514092549"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731514092549"}]},"ts":"1731514092549"} 2024-11-13T16:08:12,559 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731514091776.f8f8b8e4ae8470fd8a1d2bf0684947ef.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731514092549"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731514092549"}]},"ts":"1731514092549"} 2024-11-13T16:08:12,559 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731514091776.97998e7179d85b23a3f5928cd6511629.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731514092549"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731514092549"}]},"ts":"1731514092549"} 2024-11-13T16:08:12,559 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731514091776.519ece296a21b5ddccc38a7c928780bb.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731514092549"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731514092549"}]},"ts":"1731514092549"} 2024-11-13T16:08:12,559 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731514091776.bf1f9f9bc1bc9ffce566d41f8ae3619a.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731514092549"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731514092549"}]},"ts":"1731514092549"} 2024-11-13T16:08:12,560 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731514091776.4a146f98682f25e8cddcae5dbf318603.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731514092549"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731514092549"}]},"ts":"1731514092549"} 2024-11-13T16:08:12,560 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731514091776.6be98d350311c17b11fdb9e9a1238fca.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731514092549"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731514092549"}]},"ts":"1731514092549"} 2024-11-13T16:08:12,560 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731514091776.72d273770314688e78624993de4f3398.","families":{"info":[{"qualifier":"regioninfo","vlen":106,"tag":[],"timestamp":"1731514092549"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731514092549"}]},"ts":"1731514092549"} 2024-11-13T16:08:12,560 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731514091776.c4a87adf770b37d40ab6691e377ad1b5.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731514092549"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731514092549"}]},"ts":"1731514092549"} 2024-11-13T16:08:12,611 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-11-13T16:08:12,614 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T16:08:12,617 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731514092614"}]},"ts":"1731514092614"} 2024-11-13T16:08:12,622 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, state=ENABLING in hbase:meta 2024-11-13T16:08:12,623 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {f8f85b0bb1d8=0} racks are {/default-rack=0} 2024-11-13T16:08:12,680 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T16:08:12,680 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T16:08:12,680 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-13T16:08:12,680 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T16:08:12,680 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T16:08:12,680 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-13T16:08:12,680 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T16:08:12,680 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T16:08:12,680 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-13T16:08:12,680 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T16:08:12,684 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4aaf5667f132c289386d1b3b0c0ac57f, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8ba796d80fc3b65f0948c95ccbdad39d, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=93f8ed38b68bd013eb0cf0e7615013ae, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e72e57b96ff9bc0bbd862ce564d79c42, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=34f3d3b6a6dae1f4077567bed0bd8e75, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3945ce860c8642051e17517944a83136, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1fcf78578ccab75f1705f3c2d7bcdb14, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1a610c2403a0b491d368473a6e1fb049, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e8725a794e6d251e1bf6e3502baa7700, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fa6636be3559bc84793534420cd4f207, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=05d65f4a6bfecb3d530d15832c218252, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=78285ca993cc7fbfd78041c30558f756, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=aa124b802c708ca1f715cb1f8c9c4eaa, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3647edbd8aef23dda3887b61277da1d3, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=29df624bbcb878edce7fcdf0abceea88, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fe97fe0e9b37811ba99ca8100b461589, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7c34e9ded13254b91545186c85befbf8, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=f8f8b8e4ae8470fd8a1d2bf0684947ef, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fec5f21b48c36bf01685ec2ee58f0490, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=97998e7179d85b23a3f5928cd6511629, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=519ece296a21b5ddccc38a7c928780bb, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c4a87adf770b37d40ab6691e377ad1b5, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=bf1f9f9bc1bc9ffce566d41f8ae3619a, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4a146f98682f25e8cddcae5dbf318603, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=6be98d350311c17b11fdb9e9a1238fca, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=72d273770314688e78624993de4f3398, ASSIGN}] 2024-11-13T16:08:12,695 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=72d273770314688e78624993de4f3398, ASSIGN 2024-11-13T16:08:12,697 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=72d273770314688e78624993de4f3398, ASSIGN; state=OFFLINE, location=f8f85b0bb1d8,41333,1731514088798; forceNewPlan=false, retain=false 2024-11-13T16:08:12,699 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=6be98d350311c17b11fdb9e9a1238fca, ASSIGN 2024-11-13T16:08:12,701 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=6be98d350311c17b11fdb9e9a1238fca, ASSIGN; state=OFFLINE, location=f8f85b0bb1d8,43467,1731514088560; forceNewPlan=false, retain=false 2024-11-13T16:08:12,703 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4a146f98682f25e8cddcae5dbf318603, ASSIGN 2024-11-13T16:08:12,705 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4a146f98682f25e8cddcae5dbf318603, ASSIGN; state=OFFLINE, location=f8f85b0bb1d8,38187,1731514088725; forceNewPlan=false, retain=false 2024-11-13T16:08:12,707 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=bf1f9f9bc1bc9ffce566d41f8ae3619a, ASSIGN 2024-11-13T16:08:12,709 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=bf1f9f9bc1bc9ffce566d41f8ae3619a, ASSIGN; state=OFFLINE, location=f8f85b0bb1d8,41333,1731514088798; forceNewPlan=false, retain=false 2024-11-13T16:08:12,710 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c4a87adf770b37d40ab6691e377ad1b5, ASSIGN 2024-11-13T16:08:12,712 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c4a87adf770b37d40ab6691e377ad1b5, ASSIGN; state=OFFLINE, location=f8f85b0bb1d8,38187,1731514088725; forceNewPlan=false, retain=false 2024-11-13T16:08:12,714 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=519ece296a21b5ddccc38a7c928780bb, ASSIGN 2024-11-13T16:08:12,715 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=519ece296a21b5ddccc38a7c928780bb, ASSIGN; state=OFFLINE, location=f8f85b0bb1d8,43467,1731514088560; forceNewPlan=false, retain=false 2024-11-13T16:08:12,716 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=97998e7179d85b23a3f5928cd6511629, ASSIGN 2024-11-13T16:08:12,718 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=97998e7179d85b23a3f5928cd6511629, ASSIGN; state=OFFLINE, location=f8f85b0bb1d8,38187,1731514088725; forceNewPlan=false, retain=false 2024-11-13T16:08:12,719 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fec5f21b48c36bf01685ec2ee58f0490, ASSIGN 2024-11-13T16:08:12,720 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fec5f21b48c36bf01685ec2ee58f0490, ASSIGN; state=OFFLINE, location=f8f85b0bb1d8,43467,1731514088560; forceNewPlan=false, retain=false 2024-11-13T16:08:12,722 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=f8f8b8e4ae8470fd8a1d2bf0684947ef, ASSIGN 2024-11-13T16:08:12,723 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=f8f8b8e4ae8470fd8a1d2bf0684947ef, ASSIGN; state=OFFLINE, location=f8f85b0bb1d8,41333,1731514088798; forceNewPlan=false, retain=false 2024-11-13T16:08:12,725 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7c34e9ded13254b91545186c85befbf8, ASSIGN 2024-11-13T16:08:12,726 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7c34e9ded13254b91545186c85befbf8, ASSIGN; state=OFFLINE, location=f8f85b0bb1d8,38187,1731514088725; forceNewPlan=false, retain=false 2024-11-13T16:08:12,728 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fe97fe0e9b37811ba99ca8100b461589, ASSIGN 2024-11-13T16:08:12,730 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fe97fe0e9b37811ba99ca8100b461589, ASSIGN; state=OFFLINE, location=f8f85b0bb1d8,41333,1731514088798; forceNewPlan=false, retain=false 2024-11-13T16:08:12,732 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=29df624bbcb878edce7fcdf0abceea88, ASSIGN 2024-11-13T16:08:12,734 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=29df624bbcb878edce7fcdf0abceea88, ASSIGN; state=OFFLINE, location=f8f85b0bb1d8,43467,1731514088560; forceNewPlan=false, retain=false 2024-11-13T16:08:12,735 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3647edbd8aef23dda3887b61277da1d3, ASSIGN 2024-11-13T16:08:12,737 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3647edbd8aef23dda3887b61277da1d3, ASSIGN; state=OFFLINE, location=f8f85b0bb1d8,38187,1731514088725; forceNewPlan=false, retain=false 2024-11-13T16:08:12,740 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=aa124b802c708ca1f715cb1f8c9c4eaa, ASSIGN 2024-11-13T16:08:12,742 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=aa124b802c708ca1f715cb1f8c9c4eaa, ASSIGN; state=OFFLINE, location=f8f85b0bb1d8,43467,1731514088560; forceNewPlan=false, retain=false 2024-11-13T16:08:12,744 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=78285ca993cc7fbfd78041c30558f756, ASSIGN 2024-11-13T16:08:12,746 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=78285ca993cc7fbfd78041c30558f756, ASSIGN; state=OFFLINE, location=f8f85b0bb1d8,38187,1731514088725; forceNewPlan=false, retain=false 2024-11-13T16:08:12,748 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=05d65f4a6bfecb3d530d15832c218252, ASSIGN 2024-11-13T16:08:12,750 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=05d65f4a6bfecb3d530d15832c218252, ASSIGN; state=OFFLINE, location=f8f85b0bb1d8,41333,1731514088798; forceNewPlan=false, retain=false 2024-11-13T16:08:12,752 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fa6636be3559bc84793534420cd4f207, ASSIGN 2024-11-13T16:08:12,754 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fa6636be3559bc84793534420cd4f207, ASSIGN; state=OFFLINE, location=f8f85b0bb1d8,41333,1731514088798; forceNewPlan=false, retain=false 2024-11-13T16:08:12,756 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e8725a794e6d251e1bf6e3502baa7700, ASSIGN 2024-11-13T16:08:12,758 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e8725a794e6d251e1bf6e3502baa7700, ASSIGN; state=OFFLINE, location=f8f85b0bb1d8,43467,1731514088560; forceNewPlan=false, retain=false 2024-11-13T16:08:12,760 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1a610c2403a0b491d368473a6e1fb049, ASSIGN 2024-11-13T16:08:12,763 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1a610c2403a0b491d368473a6e1fb049, ASSIGN; state=OFFLINE, location=f8f85b0bb1d8,38187,1731514088725; forceNewPlan=false, retain=false 2024-11-13T16:08:12,766 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1fcf78578ccab75f1705f3c2d7bcdb14, ASSIGN 2024-11-13T16:08:12,769 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1fcf78578ccab75f1705f3c2d7bcdb14, ASSIGN; state=OFFLINE, location=f8f85b0bb1d8,41333,1731514088798; forceNewPlan=false, retain=false 2024-11-13T16:08:12,774 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3945ce860c8642051e17517944a83136, ASSIGN 2024-11-13T16:08:12,775 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3945ce860c8642051e17517944a83136, ASSIGN; state=OFFLINE, location=f8f85b0bb1d8,43467,1731514088560; forceNewPlan=false, retain=false 2024-11-13T16:08:12,779 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=34f3d3b6a6dae1f4077567bed0bd8e75, ASSIGN 2024-11-13T16:08:12,781 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=34f3d3b6a6dae1f4077567bed0bd8e75, ASSIGN; state=OFFLINE, location=f8f85b0bb1d8,38187,1731514088725; forceNewPlan=false, retain=false 2024-11-13T16:08:12,784 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e72e57b96ff9bc0bbd862ce564d79c42, ASSIGN 2024-11-13T16:08:12,786 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e72e57b96ff9bc0bbd862ce564d79c42, ASSIGN; state=OFFLINE, location=f8f85b0bb1d8,41333,1731514088798; forceNewPlan=false, retain=false 2024-11-13T16:08:12,789 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=93f8ed38b68bd013eb0cf0e7615013ae, ASSIGN 2024-11-13T16:08:12,791 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=93f8ed38b68bd013eb0cf0e7615013ae, ASSIGN; state=OFFLINE, location=f8f85b0bb1d8,38187,1731514088725; forceNewPlan=false, retain=false 2024-11-13T16:08:12,793 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8ba796d80fc3b65f0948c95ccbdad39d, ASSIGN 2024-11-13T16:08:12,796 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8ba796d80fc3b65f0948c95ccbdad39d, ASSIGN; state=OFFLINE, location=f8f85b0bb1d8,43467,1731514088560; forceNewPlan=false, retain=false 2024-11-13T16:08:12,798 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4aaf5667f132c289386d1b3b0c0ac57f, ASSIGN 2024-11-13T16:08:12,801 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4aaf5667f132c289386d1b3b0c0ac57f, ASSIGN; state=OFFLINE, location=f8f85b0bb1d8,41333,1731514088798; forceNewPlan=false, retain=false 2024-11-13T16:08:12,851 INFO [f8f85b0bb1d8:42949 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-11-13T16:08:12,854 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=6be98d350311c17b11fdb9e9a1238fca, regionState=OPENING, regionLocation=f8f85b0bb1d8,43467,1731514088560 2024-11-13T16:08:12,856 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=519ece296a21b5ddccc38a7c928780bb, regionState=OPENING, regionLocation=f8f85b0bb1d8,43467,1731514088560 2024-11-13T16:08:12,857 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=fec5f21b48c36bf01685ec2ee58f0490, regionState=OPENING, regionLocation=f8f85b0bb1d8,43467,1731514088560 2024-11-13T16:08:12,859 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=29df624bbcb878edce7fcdf0abceea88, regionState=OPENING, regionLocation=f8f85b0bb1d8,43467,1731514088560 2024-11-13T16:08:12,860 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=aa124b802c708ca1f715cb1f8c9c4eaa, regionState=OPENING, regionLocation=f8f85b0bb1d8,43467,1731514088560 2024-11-13T16:08:12,861 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=e8725a794e6d251e1bf6e3502baa7700, regionState=OPENING, regionLocation=f8f85b0bb1d8,43467,1731514088560 2024-11-13T16:08:12,861 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=6be98d350311c17b11fdb9e9a1238fca, ASSIGN because future has completed 2024-11-13T16:08:12,862 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=3945ce860c8642051e17517944a83136, regionState=OPENING, regionLocation=f8f85b0bb1d8,43467,1731514088560 2024-11-13T16:08:12,863 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=8ba796d80fc3b65f0948c95ccbdad39d, regionState=OPENING, regionLocation=f8f85b0bb1d8,43467,1731514088560 2024-11-13T16:08:12,864 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=72d273770314688e78624993de4f3398, regionState=OPENING, regionLocation=f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:08:12,865 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=bf1f9f9bc1bc9ffce566d41f8ae3619a, regionState=OPENING, regionLocation=f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:08:12,866 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=f8f8b8e4ae8470fd8a1d2bf0684947ef, regionState=OPENING, regionLocation=f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:08:12,868 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6be98d350311c17b11fdb9e9a1238fca, server=f8f85b0bb1d8,43467,1731514088560}] 2024-11-13T16:08:12,868 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=519ece296a21b5ddccc38a7c928780bb, ASSIGN because future has completed 2024-11-13T16:08:12,876 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fec5f21b48c36bf01685ec2ee58f0490, ASSIGN because future has completed 2024-11-13T16:08:12,877 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=29df624bbcb878edce7fcdf0abceea88, ASSIGN because future has completed 2024-11-13T16:08:12,888 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=aa124b802c708ca1f715cb1f8c9c4eaa, ASSIGN because future has completed 2024-11-13T16:08:12,895 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure aa124b802c708ca1f715cb1f8c9c4eaa, server=f8f85b0bb1d8,43467,1731514088560}] 2024-11-13T16:08:12,896 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e8725a794e6d251e1bf6e3502baa7700, ASSIGN because future has completed 2024-11-13T16:08:12,903 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3945ce860c8642051e17517944a83136, ASSIGN because future has completed 2024-11-13T16:08:12,905 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8ba796d80fc3b65f0948c95ccbdad39d, ASSIGN because future has completed 2024-11-13T16:08:12,915 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=72d273770314688e78624993de4f3398, ASSIGN because future has completed 2024-11-13T16:08:12,917 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=bf1f9f9bc1bc9ffce566d41f8ae3619a, ASSIGN because future has completed 2024-11-13T16:08:12,927 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure bf1f9f9bc1bc9ffce566d41f8ae3619a, server=f8f85b0bb1d8,41333,1731514088798}] 2024-11-13T16:08:12,928 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=f8f8b8e4ae8470fd8a1d2bf0684947ef, ASSIGN because future has completed 2024-11-13T16:08:12,934 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure f8f8b8e4ae8470fd8a1d2bf0684947ef, server=f8f85b0bb1d8,41333,1731514088798}] 2024-11-13T16:08:12,943 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 72d273770314688e78624993de4f3398, server=f8f85b0bb1d8,41333,1731514088798}] 2024-11-13T16:08:12,951 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ba796d80fc3b65f0948c95ccbdad39d, server=f8f85b0bb1d8,43467,1731514088560}] 2024-11-13T16:08:12,962 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3945ce860c8642051e17517944a83136, server=f8f85b0bb1d8,43467,1731514088560}] 2024-11-13T16:08:12,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T16:08:12,970 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure e8725a794e6d251e1bf6e3502baa7700, server=f8f85b0bb1d8,43467,1731514088560}] 2024-11-13T16:08:12,985 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 29df624bbcb878edce7fcdf0abceea88, server=f8f85b0bb1d8,43467,1731514088560}] 2024-11-13T16:08:12,993 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure fec5f21b48c36bf01685ec2ee58f0490, server=f8f85b0bb1d8,43467,1731514088560}] 2024-11-13T16:08:13,001 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 519ece296a21b5ddccc38a7c928780bb, server=f8f85b0bb1d8,43467,1731514088560}] 2024-11-13T16:08:13,011 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=fe97fe0e9b37811ba99ca8100b461589, regionState=OPENING, regionLocation=f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:08:13,012 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=05d65f4a6bfecb3d530d15832c218252, regionState=OPENING, regionLocation=f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:08:13,013 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=fa6636be3559bc84793534420cd4f207, regionState=OPENING, regionLocation=f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:08:13,013 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=1fcf78578ccab75f1705f3c2d7bcdb14, regionState=OPENING, regionLocation=f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:08:13,014 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=e72e57b96ff9bc0bbd862ce564d79c42, regionState=OPENING, regionLocation=f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:08:13,015 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4aaf5667f132c289386d1b3b0c0ac57f, regionState=OPENING, regionLocation=f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:08:13,015 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fe97fe0e9b37811ba99ca8100b461589, ASSIGN because future has completed 2024-11-13T16:08:13,015 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=4a146f98682f25e8cddcae5dbf318603, regionState=OPENING, regionLocation=f8f85b0bb1d8,38187,1731514088725 2024-11-13T16:08:13,016 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure fe97fe0e9b37811ba99ca8100b461589, server=f8f85b0bb1d8,41333,1731514088798}] 2024-11-13T16:08:13,018 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=05d65f4a6bfecb3d530d15832c218252, ASSIGN because future has completed 2024-11-13T16:08:13,021 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fa6636be3559bc84793534420cd4f207, ASSIGN because future has completed 2024-11-13T16:08:13,022 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1fcf78578ccab75f1705f3c2d7bcdb14, ASSIGN because future has completed 2024-11-13T16:08:13,030 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e72e57b96ff9bc0bbd862ce564d79c42, ASSIGN because future has completed 2024-11-13T16:08:13,031 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure e72e57b96ff9bc0bbd862ce564d79c42, server=f8f85b0bb1d8,41333,1731514088798}] 2024-11-13T16:08:13,040 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4aaf5667f132c289386d1b3b0c0ac57f, ASSIGN because future has completed 2024-11-13T16:08:13,041 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4aaf5667f132c289386d1b3b0c0ac57f, server=f8f85b0bb1d8,41333,1731514088798}] 2024-11-13T16:08:13,043 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4a146f98682f25e8cddcae5dbf318603, ASSIGN because future has completed 2024-11-13T16:08:13,055 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731514091776.3945ce860c8642051e17517944a83136. 2024-11-13T16:08:13,055 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 3945ce860c8642051e17517944a83136, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731514091776.3945ce860c8642051e17517944a83136.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-13T16:08:13,056 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 3945ce860c8642051e17517944a83136 2024-11-13T16:08:13,056 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731514091776.3945ce860c8642051e17517944a83136.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:13,058 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 3945ce860c8642051e17517944a83136 2024-11-13T16:08:13,058 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 3945ce860c8642051e17517944a83136 2024-11-13T16:08:13,063 INFO [StoreOpener-3945ce860c8642051e17517944a83136-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 3945ce860c8642051e17517944a83136 2024-11-13T16:08:13,066 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4a146f98682f25e8cddcae5dbf318603, server=f8f85b0bb1d8,38187,1731514088725}] 2024-11-13T16:08:13,066 INFO [StoreOpener-3945ce860c8642051e17517944a83136-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3945ce860c8642051e17517944a83136 columnFamilyName colfamily11 2024-11-13T16:08:13,066 DEBUG [StoreOpener-3945ce860c8642051e17517944a83136-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,068 INFO [StoreOpener-3945ce860c8642051e17517944a83136-1 {}] regionserver.HStore(327): Store=3945ce860c8642051e17517944a83136/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,068 INFO [StoreOpener-3945ce860c8642051e17517944a83136-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 3945ce860c8642051e17517944a83136 2024-11-13T16:08:13,071 INFO [StoreOpener-3945ce860c8642051e17517944a83136-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3945ce860c8642051e17517944a83136 columnFamilyName colfamily21 2024-11-13T16:08:13,071 DEBUG [StoreOpener-3945ce860c8642051e17517944a83136-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,072 INFO [StoreOpener-3945ce860c8642051e17517944a83136-1 {}] regionserver.HStore(327): Store=3945ce860c8642051e17517944a83136/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,073 INFO [StoreOpener-3945ce860c8642051e17517944a83136-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 3945ce860c8642051e17517944a83136 2024-11-13T16:08:13,079 INFO [StoreOpener-3945ce860c8642051e17517944a83136-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3945ce860c8642051e17517944a83136 columnFamilyName colfamily31 2024-11-13T16:08:13,079 DEBUG [StoreOpener-3945ce860c8642051e17517944a83136-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,081 INFO [StoreOpener-3945ce860c8642051e17517944a83136-1 {}] regionserver.HStore(327): Store=3945ce860c8642051e17517944a83136/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,082 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 3945ce860c8642051e17517944a83136 2024-11-13T16:08:13,087 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3945ce860c8642051e17517944a83136 2024-11-13T16:08:13,090 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T16:08:13,091 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3945ce860c8642051e17517944a83136 2024-11-13T16:08:13,091 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1fcf78578ccab75f1705f3c2d7bcdb14, server=f8f85b0bb1d8,41333,1731514088798}] 2024-11-13T16:08:13,092 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51525, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T16:08:13,106 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 3945ce860c8642051e17517944a83136 2024-11-13T16:08:13,106 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 3945ce860c8642051e17517944a83136 2024-11-13T16:08:13,106 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731514091776.f8f8b8e4ae8470fd8a1d2bf0684947ef. 2024-11-13T16:08:13,106 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa6636be3559bc84793534420cd4f207, server=f8f85b0bb1d8,41333,1731514088798}] 2024-11-13T16:08:13,106 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => f8f8b8e4ae8470fd8a1d2bf0684947ef, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731514091776.f8f8b8e4ae8470fd8a1d2bf0684947ef.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-13T16:08:13,107 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true f8f8b8e4ae8470fd8a1d2bf0684947ef 2024-11-13T16:08:13,107 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731514091776.f8f8b8e4ae8470fd8a1d2bf0684947ef.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:13,107 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for f8f8b8e4ae8470fd8a1d2bf0684947ef 2024-11-13T16:08:13,107 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for f8f8b8e4ae8470fd8a1d2bf0684947ef 2024-11-13T16:08:13,107 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T16:08:13,113 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 3945ce860c8642051e17517944a83136 2024-11-13T16:08:13,121 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 05d65f4a6bfecb3d530d15832c218252, server=f8f85b0bb1d8,41333,1731514088798}] 2024-11-13T16:08:13,127 INFO [StoreOpener-f8f8b8e4ae8470fd8a1d2bf0684947ef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region f8f8b8e4ae8470fd8a1d2bf0684947ef 2024-11-13T16:08:13,132 INFO [StoreOpener-f8f8b8e4ae8470fd8a1d2bf0684947ef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f8f8b8e4ae8470fd8a1d2bf0684947ef columnFamilyName colfamily11 2024-11-13T16:08:13,132 DEBUG [StoreOpener-f8f8b8e4ae8470fd8a1d2bf0684947ef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,133 INFO [StoreOpener-f8f8b8e4ae8470fd8a1d2bf0684947ef-1 {}] regionserver.HStore(327): Store=f8f8b8e4ae8470fd8a1d2bf0684947ef/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,133 INFO [StoreOpener-f8f8b8e4ae8470fd8a1d2bf0684947ef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region f8f8b8e4ae8470fd8a1d2bf0684947ef 2024-11-13T16:08:13,135 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=c4a87adf770b37d40ab6691e377ad1b5, regionState=OPENING, regionLocation=f8f85b0bb1d8,38187,1731514088725 2024-11-13T16:08:13,136 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=97998e7179d85b23a3f5928cd6511629, regionState=OPENING, regionLocation=f8f85b0bb1d8,38187,1731514088725 2024-11-13T16:08:13,138 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=7c34e9ded13254b91545186c85befbf8, regionState=OPENING, regionLocation=f8f85b0bb1d8,38187,1731514088725 2024-11-13T16:08:13,138 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=3647edbd8aef23dda3887b61277da1d3, regionState=OPENING, regionLocation=f8f85b0bb1d8,38187,1731514088725 2024-11-13T16:08:13,139 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c4a87adf770b37d40ab6691e377ad1b5, ASSIGN because future has completed 2024-11-13T16:08:13,139 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3945ce860c8642051e17517944a83136/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T16:08:13,139 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure c4a87adf770b37d40ab6691e377ad1b5, server=f8f85b0bb1d8,38187,1731514088725}] 2024-11-13T16:08:13,140 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 3945ce860c8642051e17517944a83136; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63244207, jitterRate=-0.0575878769159317}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T16:08:13,140 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3945ce860c8642051e17517944a83136 2024-11-13T16:08:13,141 INFO [StoreOpener-f8f8b8e4ae8470fd8a1d2bf0684947ef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f8f8b8e4ae8470fd8a1d2bf0684947ef columnFamilyName colfamily21 2024-11-13T16:08:13,141 DEBUG [StoreOpener-f8f8b8e4ae8470fd8a1d2bf0684947ef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,141 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 3945ce860c8642051e17517944a83136: Running coprocessor pre-open hook at 1731514093059Writing region info on filesystem at 1731514093059Initializing all the Stores at 1731514093061 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093061Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093062 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093063 (+1 ms)Cleaning up temporary data from old regions at 1731514093106 (+43 ms)Running coprocessor post-open hooks at 1731514093140 (+34 ms)Region opened successfully at 1731514093141 (+1 ms) 2024-11-13T16:08:13,143 INFO [StoreOpener-f8f8b8e4ae8470fd8a1d2bf0684947ef-1 {}] regionserver.HStore(327): Store=f8f8b8e4ae8470fd8a1d2bf0684947ef/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,143 INFO [StoreOpener-f8f8b8e4ae8470fd8a1d2bf0684947ef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region f8f8b8e4ae8470fd8a1d2bf0684947ef 2024-11-13T16:08:13,146 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731514091776.3945ce860c8642051e17517944a83136., pid=37, masterSystemTime=1731514093033 2024-11-13T16:08:13,148 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=97998e7179d85b23a3f5928cd6511629, ASSIGN because future has completed 2024-11-13T16:08:13,151 INFO [StoreOpener-f8f8b8e4ae8470fd8a1d2bf0684947ef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f8f8b8e4ae8470fd8a1d2bf0684947ef columnFamilyName colfamily31 2024-11-13T16:08:13,151 DEBUG [StoreOpener-f8f8b8e4ae8470fd8a1d2bf0684947ef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,153 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7c34e9ded13254b91545186c85befbf8, ASSIGN because future has completed 2024-11-13T16:08:13,156 INFO [StoreOpener-f8f8b8e4ae8470fd8a1d2bf0684947ef-1 {}] regionserver.HStore(327): Store=f8f8b8e4ae8470fd8a1d2bf0684947ef/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,156 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731514091776.3945ce860c8642051e17517944a83136. 2024-11-13T16:08:13,156 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731514091776.3945ce860c8642051e17517944a83136. 2024-11-13T16:08:13,156 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for f8f8b8e4ae8470fd8a1d2bf0684947ef 2024-11-13T16:08:13,156 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731514091776.6be98d350311c17b11fdb9e9a1238fca. 2024-11-13T16:08:13,156 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => 6be98d350311c17b11fdb9e9a1238fca, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731514091776.6be98d350311c17b11fdb9e9a1238fca.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-13T16:08:13,157 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3647edbd8aef23dda3887b61277da1d3, ASSIGN because future has completed 2024-11-13T16:08:13,157 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 6be98d350311c17b11fdb9e9a1238fca 2024-11-13T16:08:13,157 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731514091776.6be98d350311c17b11fdb9e9a1238fca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:13,157 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for 6be98d350311c17b11fdb9e9a1238fca 2024-11-13T16:08:13,157 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for 6be98d350311c17b11fdb9e9a1238fca 2024-11-13T16:08:13,157 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f8f8b8e4ae8470fd8a1d2bf0684947ef 2024-11-13T16:08:13,158 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f8f8b8e4ae8470fd8a1d2bf0684947ef 2024-11-13T16:08:13,161 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3647edbd8aef23dda3887b61277da1d3, server=f8f85b0bb1d8,38187,1731514088725}] 2024-11-13T16:08:13,163 INFO [StoreOpener-6be98d350311c17b11fdb9e9a1238fca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 6be98d350311c17b11fdb9e9a1238fca 2024-11-13T16:08:13,167 INFO [StoreOpener-6be98d350311c17b11fdb9e9a1238fca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6be98d350311c17b11fdb9e9a1238fca columnFamilyName colfamily11 2024-11-13T16:08:13,168 DEBUG [StoreOpener-6be98d350311c17b11fdb9e9a1238fca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,168 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for f8f8b8e4ae8470fd8a1d2bf0684947ef 2024-11-13T16:08:13,168 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for f8f8b8e4ae8470fd8a1d2bf0684947ef 2024-11-13T16:08:13,169 INFO [StoreOpener-6be98d350311c17b11fdb9e9a1238fca-1 {}] regionserver.HStore(327): Store=6be98d350311c17b11fdb9e9a1238fca/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,169 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T16:08:13,169 INFO [StoreOpener-6be98d350311c17b11fdb9e9a1238fca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 6be98d350311c17b11fdb9e9a1238fca 2024-11-13T16:08:13,172 INFO [StoreOpener-6be98d350311c17b11fdb9e9a1238fca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6be98d350311c17b11fdb9e9a1238fca columnFamilyName colfamily21 2024-11-13T16:08:13,173 DEBUG [StoreOpener-6be98d350311c17b11fdb9e9a1238fca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,174 INFO [StoreOpener-6be98d350311c17b11fdb9e9a1238fca-1 {}] regionserver.HStore(327): Store=6be98d350311c17b11fdb9e9a1238fca/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,175 INFO [StoreOpener-6be98d350311c17b11fdb9e9a1238fca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 6be98d350311c17b11fdb9e9a1238fca 2024-11-13T16:08:13,175 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for f8f8b8e4ae8470fd8a1d2bf0684947ef 2024-11-13T16:08:13,176 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=3945ce860c8642051e17517944a83136, regionState=OPEN, openSeqNum=2, regionLocation=f8f85b0bb1d8,43467,1731514088560 2024-11-13T16:08:13,178 INFO [StoreOpener-6be98d350311c17b11fdb9e9a1238fca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6be98d350311c17b11fdb9e9a1238fca columnFamilyName colfamily31 2024-11-13T16:08:13,178 DEBUG [StoreOpener-6be98d350311c17b11fdb9e9a1238fca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,180 INFO [StoreOpener-6be98d350311c17b11fdb9e9a1238fca-1 {}] regionserver.HStore(327): Store=6be98d350311c17b11fdb9e9a1238fca/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,180 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for 6be98d350311c17b11fdb9e9a1238fca 2024-11-13T16:08:13,181 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3945ce860c8642051e17517944a83136, server=f8f85b0bb1d8,43467,1731514088560 because future has completed 2024-11-13T16:08:13,181 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6be98d350311c17b11fdb9e9a1238fca 2024-11-13T16:08:13,182 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7c34e9ded13254b91545186c85befbf8, server=f8f85b0bb1d8,38187,1731514088725}] 2024-11-13T16:08:13,182 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6be98d350311c17b11fdb9e9a1238fca 2024-11-13T16:08:13,185 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for 6be98d350311c17b11fdb9e9a1238fca 2024-11-13T16:08:13,185 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for 6be98d350311c17b11fdb9e9a1238fca 2024-11-13T16:08:13,185 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f8f8b8e4ae8470fd8a1d2bf0684947ef/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T16:08:13,186 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T16:08:13,186 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened f8f8b8e4ae8470fd8a1d2bf0684947ef; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61197821, jitterRate=-0.08808140456676483}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T16:08:13,187 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f8f8b8e4ae8470fd8a1d2bf0684947ef 2024-11-13T16:08:13,187 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for f8f8b8e4ae8470fd8a1d2bf0684947ef: Running coprocessor pre-open hook at 1731514093107Writing region info on filesystem at 1731514093107Initializing all the Stores at 1731514093123 (+16 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093123Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093126 (+3 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093126Cleaning up temporary data from old regions at 1731514093168 (+42 ms)Running coprocessor post-open hooks at 1731514093187 (+19 ms)Region opened successfully at 1731514093187 2024-11-13T16:08:13,191 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for 6be98d350311c17b11fdb9e9a1238fca 2024-11-13T16:08:13,191 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731514091776.f8f8b8e4ae8470fd8a1d2bf0684947ef., pid=34, masterSystemTime=1731514093089 2024-11-13T16:08:13,195 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731514091776.f8f8b8e4ae8470fd8a1d2bf0684947ef. 2024-11-13T16:08:13,195 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731514091776.f8f8b8e4ae8470fd8a1d2bf0684947ef. 2024-11-13T16:08:13,195 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731514091776.e72e57b96ff9bc0bbd862ce564d79c42. 2024-11-13T16:08:13,196 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => e72e57b96ff9bc0bbd862ce564d79c42, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731514091776.e72e57b96ff9bc0bbd862ce564d79c42.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-13T16:08:13,196 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true e72e57b96ff9bc0bbd862ce564d79c42 2024-11-13T16:08:13,197 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731514091776.e72e57b96ff9bc0bbd862ce564d79c42.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:13,197 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for e72e57b96ff9bc0bbd862ce564d79c42 2024-11-13T16:08:13,197 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for e72e57b96ff9bc0bbd862ce564d79c42 2024-11-13T16:08:13,202 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=10 2024-11-13T16:08:13,202 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 3945ce860c8642051e17517944a83136, server=f8f85b0bb1d8,43467,1731514088560 in 232 msec 2024-11-13T16:08:13,203 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6be98d350311c17b11fdb9e9a1238fca/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T16:08:13,204 INFO [StoreOpener-e72e57b96ff9bc0bbd862ce564d79c42-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region e72e57b96ff9bc0bbd862ce564d79c42 2024-11-13T16:08:13,205 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened 6be98d350311c17b11fdb9e9a1238fca; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63640149, jitterRate=-0.05168788135051727}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T16:08:13,206 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6be98d350311c17b11fdb9e9a1238fca 2024-11-13T16:08:13,206 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for 6be98d350311c17b11fdb9e9a1238fca: Running coprocessor pre-open hook at 1731514093157Writing region info on filesystem at 1731514093158 (+1 ms)Initializing all the Stores at 1731514093161 (+3 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093161Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093162 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093162Cleaning up temporary data from old regions at 1731514093185 (+23 ms)Running coprocessor post-open hooks at 1731514093206 (+21 ms)Region opened successfully at 1731514093206 2024-11-13T16:08:13,208 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731514091776.6be98d350311c17b11fdb9e9a1238fca., pid=31, masterSystemTime=1731514093033 2024-11-13T16:08:13,210 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3945ce860c8642051e17517944a83136, ASSIGN in 519 msec 2024-11-13T16:08:13,212 INFO [StoreOpener-e72e57b96ff9bc0bbd862ce564d79c42-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e72e57b96ff9bc0bbd862ce564d79c42 columnFamilyName colfamily11 2024-11-13T16:08:13,212 DEBUG [StoreOpener-e72e57b96ff9bc0bbd862ce564d79c42-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,217 INFO [StoreOpener-e72e57b96ff9bc0bbd862ce564d79c42-1 {}] regionserver.HStore(327): Store=e72e57b96ff9bc0bbd862ce564d79c42/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,218 INFO [StoreOpener-e72e57b96ff9bc0bbd862ce564d79c42-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region e72e57b96ff9bc0bbd862ce564d79c42 2024-11-13T16:08:13,221 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=f8f8b8e4ae8470fd8a1d2bf0684947ef, regionState=OPEN, openSeqNum=2, regionLocation=f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:08:13,222 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731514091776.6be98d350311c17b11fdb9e9a1238fca. 2024-11-13T16:08:13,223 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731514091776.6be98d350311c17b11fdb9e9a1238fca. 2024-11-13T16:08:13,224 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731514091776.aa124b802c708ca1f715cb1f8c9c4eaa. 2024-11-13T16:08:13,227 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T16:08:13,224 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => aa124b802c708ca1f715cb1f8c9c4eaa, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731514091776.aa124b802c708ca1f715cb1f8c9c4eaa.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-13T16:08:13,228 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true aa124b802c708ca1f715cb1f8c9c4eaa 2024-11-13T16:08:13,229 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731514091776.aa124b802c708ca1f715cb1f8c9c4eaa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:13,229 INFO [StoreOpener-e72e57b96ff9bc0bbd862ce564d79c42-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e72e57b96ff9bc0bbd862ce564d79c42 columnFamilyName colfamily21 2024-11-13T16:08:13,229 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for aa124b802c708ca1f715cb1f8c9c4eaa 2024-11-13T16:08:13,229 DEBUG [StoreOpener-e72e57b96ff9bc0bbd862ce564d79c42-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,229 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for aa124b802c708ca1f715cb1f8c9c4eaa 2024-11-13T16:08:13,231 INFO [StoreOpener-e72e57b96ff9bc0bbd862ce564d79c42-1 {}] regionserver.HStore(327): Store=e72e57b96ff9bc0bbd862ce564d79c42/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,231 INFO [StoreOpener-e72e57b96ff9bc0bbd862ce564d79c42-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region e72e57b96ff9bc0bbd862ce564d79c42 2024-11-13T16:08:13,234 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure f8f8b8e4ae8470fd8a1d2bf0684947ef, server=f8f85b0bb1d8,41333,1731514088798 because future has completed 2024-11-13T16:08:13,237 INFO [StoreOpener-e72e57b96ff9bc0bbd862ce564d79c42-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e72e57b96ff9bc0bbd862ce564d79c42 columnFamilyName colfamily31 2024-11-13T16:08:13,238 DEBUG [StoreOpener-e72e57b96ff9bc0bbd862ce564d79c42-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,239 INFO [StoreOpener-e72e57b96ff9bc0bbd862ce564d79c42-1 {}] regionserver.HStore(327): Store=e72e57b96ff9bc0bbd862ce564d79c42/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,239 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for e72e57b96ff9bc0bbd862ce564d79c42 2024-11-13T16:08:13,240 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49947, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T16:08:13,243 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=6be98d350311c17b11fdb9e9a1238fca, regionState=OPEN, openSeqNum=2, regionLocation=f8f85b0bb1d8,43467,1731514088560 2024-11-13T16:08:13,246 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e72e57b96ff9bc0bbd862ce564d79c42 2024-11-13T16:08:13,247 INFO [StoreOpener-aa124b802c708ca1f715cb1f8c9c4eaa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region aa124b802c708ca1f715cb1f8c9c4eaa 2024-11-13T16:08:13,250 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e72e57b96ff9bc0bbd862ce564d79c42 2024-11-13T16:08:13,250 INFO [StoreOpener-aa124b802c708ca1f715cb1f8c9c4eaa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aa124b802c708ca1f715cb1f8c9c4eaa columnFamilyName colfamily11 2024-11-13T16:08:13,251 DEBUG [StoreOpener-aa124b802c708ca1f715cb1f8c9c4eaa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,254 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6be98d350311c17b11fdb9e9a1238fca, server=f8f85b0bb1d8,43467,1731514088560 because future has completed 2024-11-13T16:08:13,256 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731514091776.3647edbd8aef23dda3887b61277da1d3. 2024-11-13T16:08:13,256 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => 3647edbd8aef23dda3887b61277da1d3, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731514091776.3647edbd8aef23dda3887b61277da1d3.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-13T16:08:13,256 INFO [StoreOpener-aa124b802c708ca1f715cb1f8c9c4eaa-1 {}] regionserver.HStore(327): Store=aa124b802c708ca1f715cb1f8c9c4eaa/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,256 INFO [StoreOpener-aa124b802c708ca1f715cb1f8c9c4eaa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region aa124b802c708ca1f715cb1f8c9c4eaa 2024-11-13T16:08:13,257 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 3647edbd8aef23dda3887b61277da1d3 2024-11-13T16:08:13,257 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731514091776.3647edbd8aef23dda3887b61277da1d3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:13,257 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for 3647edbd8aef23dda3887b61277da1d3 2024-11-13T16:08:13,259 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for e72e57b96ff9bc0bbd862ce564d79c42 2024-11-13T16:08:13,259 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for e72e57b96ff9bc0bbd862ce564d79c42 2024-11-13T16:08:13,261 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for 3647edbd8aef23dda3887b61277da1d3 2024-11-13T16:08:13,262 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T16:08:13,268 INFO [StoreOpener-aa124b802c708ca1f715cb1f8c9c4eaa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aa124b802c708ca1f715cb1f8c9c4eaa columnFamilyName colfamily21 2024-11-13T16:08:13,269 DEBUG [StoreOpener-aa124b802c708ca1f715cb1f8c9c4eaa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,268 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for e72e57b96ff9bc0bbd862ce564d79c42 2024-11-13T16:08:13,272 INFO [StoreOpener-aa124b802c708ca1f715cb1f8c9c4eaa-1 {}] regionserver.HStore(327): Store=aa124b802c708ca1f715cb1f8c9c4eaa/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,272 INFO [StoreOpener-aa124b802c708ca1f715cb1f8c9c4eaa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region aa124b802c708ca1f715cb1f8c9c4eaa 2024-11-13T16:08:13,275 INFO [StoreOpener-aa124b802c708ca1f715cb1f8c9c4eaa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aa124b802c708ca1f715cb1f8c9c4eaa columnFamilyName colfamily31 2024-11-13T16:08:13,275 DEBUG [StoreOpener-aa124b802c708ca1f715cb1f8c9c4eaa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,276 INFO [StoreOpener-aa124b802c708ca1f715cb1f8c9c4eaa-1 {}] regionserver.HStore(327): Store=aa124b802c708ca1f715cb1f8c9c4eaa/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,276 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42949 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=f8f85b0bb1d8,43467,1731514088560, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=6be98d350311c17b11fdb9e9a1238fca. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T16:08:13,276 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for aa124b802c708ca1f715cb1f8c9c4eaa 2024-11-13T16:08:13,277 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aa124b802c708ca1f715cb1f8c9c4eaa 2024-11-13T16:08:13,277 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aa124b802c708ca1f715cb1f8c9c4eaa 2024-11-13T16:08:13,279 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for aa124b802c708ca1f715cb1f8c9c4eaa 2024-11-13T16:08:13,279 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for aa124b802c708ca1f715cb1f8c9c4eaa 2024-11-13T16:08:13,280 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T16:08:13,282 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=22 2024-11-13T16:08:13,282 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure f8f8b8e4ae8470fd8a1d2bf0684947ef, server=f8f85b0bb1d8,41333,1731514088798 in 322 msec 2024-11-13T16:08:13,283 INFO [StoreOpener-3647edbd8aef23dda3887b61277da1d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 3647edbd8aef23dda3887b61277da1d3 2024-11-13T16:08:13,286 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for aa124b802c708ca1f715cb1f8c9c4eaa 2024-11-13T16:08:13,288 INFO [StoreOpener-3647edbd8aef23dda3887b61277da1d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3647edbd8aef23dda3887b61277da1d3 columnFamilyName colfamily11 2024-11-13T16:08:13,288 DEBUG [StoreOpener-3647edbd8aef23dda3887b61277da1d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,288 INFO [StoreOpener-3647edbd8aef23dda3887b61277da1d3-1 {}] regionserver.HStore(327): Store=3647edbd8aef23dda3887b61277da1d3/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,289 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=f8f8b8e4ae8470fd8a1d2bf0684947ef, ASSIGN in 598 msec 2024-11-13T16:08:13,289 INFO [StoreOpener-3647edbd8aef23dda3887b61277da1d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 3647edbd8aef23dda3887b61277da1d3 2024-11-13T16:08:13,291 INFO [StoreOpener-3647edbd8aef23dda3887b61277da1d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3647edbd8aef23dda3887b61277da1d3 columnFamilyName colfamily21 2024-11-13T16:08:13,291 DEBUG [StoreOpener-3647edbd8aef23dda3887b61277da1d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,292 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e72e57b96ff9bc0bbd862ce564d79c42/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T16:08:13,293 INFO [StoreOpener-3647edbd8aef23dda3887b61277da1d3-1 {}] regionserver.HStore(327): Store=3647edbd8aef23dda3887b61277da1d3/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,293 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened e72e57b96ff9bc0bbd862ce564d79c42; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65866930, jitterRate=-0.018506258726119995}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T16:08:13,293 INFO [StoreOpener-3647edbd8aef23dda3887b61277da1d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 3647edbd8aef23dda3887b61277da1d3 2024-11-13T16:08:13,293 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e72e57b96ff9bc0bbd862ce564d79c42 2024-11-13T16:08:13,293 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for e72e57b96ff9bc0bbd862ce564d79c42: Running coprocessor pre-open hook at 1731514093197Writing region info on filesystem at 1731514093197Initializing all the Stores at 1731514093200 (+3 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093200Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093204 (+4 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093204Cleaning up temporary data from old regions at 1731514093259 (+55 ms)Running coprocessor post-open hooks at 1731514093293 (+34 ms)Region opened successfully at 1731514093293 2024-11-13T16:08:13,296 INFO [StoreOpener-3647edbd8aef23dda3887b61277da1d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3647edbd8aef23dda3887b61277da1d3 columnFamilyName colfamily31 2024-11-13T16:08:13,296 DEBUG [StoreOpener-3647edbd8aef23dda3887b61277da1d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,297 INFO [StoreOpener-3647edbd8aef23dda3887b61277da1d3-1 {}] regionserver.HStore(327): Store=3647edbd8aef23dda3887b61277da1d3/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,297 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for 3647edbd8aef23dda3887b61277da1d3 2024-11-13T16:08:13,298 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731514091776.e72e57b96ff9bc0bbd862ce564d79c42., pid=43, masterSystemTime=1731514093089 2024-11-13T16:08:13,298 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3647edbd8aef23dda3887b61277da1d3 2024-11-13T16:08:13,299 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3647edbd8aef23dda3887b61277da1d3 2024-11-13T16:08:13,301 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=29 2024-11-13T16:08:13,301 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 6be98d350311c17b11fdb9e9a1238fca, server=f8f85b0bb1d8,43467,1731514088560 in 423 msec 2024-11-13T16:08:13,301 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for 3647edbd8aef23dda3887b61277da1d3 2024-11-13T16:08:13,302 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for 3647edbd8aef23dda3887b61277da1d3 2024-11-13T16:08:13,303 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T16:08:13,305 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731514091776.e72e57b96ff9bc0bbd862ce564d79c42. 2024-11-13T16:08:13,305 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731514091776.e72e57b96ff9bc0bbd862ce564d79c42. 2024-11-13T16:08:13,305 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731514091776.bf1f9f9bc1bc9ffce566d41f8ae3619a. 2024-11-13T16:08:13,305 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => bf1f9f9bc1bc9ffce566d41f8ae3619a, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731514091776.bf1f9f9bc1bc9ffce566d41f8ae3619a.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-13T16:08:13,306 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true bf1f9f9bc1bc9ffce566d41f8ae3619a 2024-11-13T16:08:13,306 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731514091776.bf1f9f9bc1bc9ffce566d41f8ae3619a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:13,306 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for bf1f9f9bc1bc9ffce566d41f8ae3619a 2024-11-13T16:08:13,306 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for bf1f9f9bc1bc9ffce566d41f8ae3619a 2024-11-13T16:08:13,306 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for 3647edbd8aef23dda3887b61277da1d3 2024-11-13T16:08:13,309 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=6be98d350311c17b11fdb9e9a1238fca, ASSIGN in 618 msec 2024-11-13T16:08:13,310 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=e72e57b96ff9bc0bbd862ce564d79c42, regionState=OPEN, openSeqNum=2, regionLocation=f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:08:13,312 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aa124b802c708ca1f715cb1f8c9c4eaa/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T16:08:13,312 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 97998e7179d85b23a3f5928cd6511629, server=f8f85b0bb1d8,38187,1731514088725}] 2024-11-13T16:08:13,313 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened aa124b802c708ca1f715cb1f8c9c4eaa; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63115734, jitterRate=-0.05950227379798889}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T16:08:13,314 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for aa124b802c708ca1f715cb1f8c9c4eaa 2024-11-13T16:08:13,314 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for aa124b802c708ca1f715cb1f8c9c4eaa: Running coprocessor pre-open hook at 1731514093229Writing region info on filesystem at 1731514093229Initializing all the Stores at 1731514093235 (+6 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093235Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093246 (+11 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093246Cleaning up temporary data from old regions at 1731514093279 (+33 ms)Running coprocessor post-open hooks at 1731514093314 (+35 ms)Region opened successfully at 1731514093314 2024-11-13T16:08:13,317 INFO [StoreOpener-bf1f9f9bc1bc9ffce566d41f8ae3619a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region bf1f9f9bc1bc9ffce566d41f8ae3619a 2024-11-13T16:08:13,318 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3647edbd8aef23dda3887b61277da1d3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T16:08:13,320 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731514091776.aa124b802c708ca1f715cb1f8c9c4eaa., pid=32, masterSystemTime=1731514093033 2024-11-13T16:08:13,320 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened 3647edbd8aef23dda3887b61277da1d3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73212211, jitterRate=0.09094695746898651}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T16:08:13,320 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3647edbd8aef23dda3887b61277da1d3 2024-11-13T16:08:13,321 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for 3647edbd8aef23dda3887b61277da1d3: Running coprocessor pre-open hook at 1731514093261Writing region info on filesystem at 1731514093261Initializing all the Stores at 1731514093268 (+7 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093268Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093282 (+14 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093282Cleaning up temporary data from old regions at 1731514093302 (+20 ms)Running coprocessor post-open hooks at 1731514093320 (+18 ms)Region opened successfully at 1731514093320 2024-11-13T16:08:13,321 INFO [StoreOpener-bf1f9f9bc1bc9ffce566d41f8ae3619a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bf1f9f9bc1bc9ffce566d41f8ae3619a columnFamilyName colfamily11 2024-11-13T16:08:13,321 DEBUG [StoreOpener-bf1f9f9bc1bc9ffce566d41f8ae3619a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,322 INFO [StoreOpener-bf1f9f9bc1bc9ffce566d41f8ae3619a-1 {}] regionserver.HStore(327): Store=bf1f9f9bc1bc9ffce566d41f8ae3619a/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,323 INFO [StoreOpener-bf1f9f9bc1bc9ffce566d41f8ae3619a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region bf1f9f9bc1bc9ffce566d41f8ae3619a 2024-11-13T16:08:13,323 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure e72e57b96ff9bc0bbd862ce564d79c42, server=f8f85b0bb1d8,41333,1731514088798 because future has completed 2024-11-13T16:08:13,323 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731514091776.3647edbd8aef23dda3887b61277da1d3., pid=50, masterSystemTime=1731514093227 2024-11-13T16:08:13,326 INFO [StoreOpener-bf1f9f9bc1bc9ffce566d41f8ae3619a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bf1f9f9bc1bc9ffce566d41f8ae3619a columnFamilyName colfamily21 2024-11-13T16:08:13,326 DEBUG [StoreOpener-bf1f9f9bc1bc9ffce566d41f8ae3619a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,327 INFO [StoreOpener-bf1f9f9bc1bc9ffce566d41f8ae3619a-1 {}] regionserver.HStore(327): Store=bf1f9f9bc1bc9ffce566d41f8ae3619a/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,328 INFO [StoreOpener-bf1f9f9bc1bc9ffce566d41f8ae3619a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region bf1f9f9bc1bc9ffce566d41f8ae3619a 2024-11-13T16:08:13,330 INFO [StoreOpener-bf1f9f9bc1bc9ffce566d41f8ae3619a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bf1f9f9bc1bc9ffce566d41f8ae3619a columnFamilyName colfamily31 2024-11-13T16:08:13,331 DEBUG [StoreOpener-bf1f9f9bc1bc9ffce566d41f8ae3619a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,331 INFO [StoreOpener-bf1f9f9bc1bc9ffce566d41f8ae3619a-1 {}] regionserver.HStore(327): Store=bf1f9f9bc1bc9ffce566d41f8ae3619a/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,332 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for bf1f9f9bc1bc9ffce566d41f8ae3619a 2024-11-13T16:08:13,333 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bf1f9f9bc1bc9ffce566d41f8ae3619a 2024-11-13T16:08:13,333 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731514091776.aa124b802c708ca1f715cb1f8c9c4eaa. 2024-11-13T16:08:13,334 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731514091776.aa124b802c708ca1f715cb1f8c9c4eaa. 2024-11-13T16:08:13,334 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bf1f9f9bc1bc9ffce566d41f8ae3619a 2024-11-13T16:08:13,334 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731514091776.29df624bbcb878edce7fcdf0abceea88. 2024-11-13T16:08:13,334 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 29df624bbcb878edce7fcdf0abceea88, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731514091776.29df624bbcb878edce7fcdf0abceea88.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-13T16:08:13,335 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 29df624bbcb878edce7fcdf0abceea88 2024-11-13T16:08:13,335 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731514091776.3647edbd8aef23dda3887b61277da1d3. 2024-11-13T16:08:13,335 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731514091776.29df624bbcb878edce7fcdf0abceea88.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:13,335 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731514091776.3647edbd8aef23dda3887b61277da1d3. 2024-11-13T16:08:13,335 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 29df624bbcb878edce7fcdf0abceea88 2024-11-13T16:08:13,335 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731514091776.7c34e9ded13254b91545186c85befbf8. 2024-11-13T16:08:13,335 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 29df624bbcb878edce7fcdf0abceea88 2024-11-13T16:08:13,335 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => 7c34e9ded13254b91545186c85befbf8, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731514091776.7c34e9ded13254b91545186c85befbf8.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-13T16:08:13,336 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 7c34e9ded13254b91545186c85befbf8 2024-11-13T16:08:13,336 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731514091776.7c34e9ded13254b91545186c85befbf8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:13,336 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for 7c34e9ded13254b91545186c85befbf8 2024-11-13T16:08:13,336 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for 7c34e9ded13254b91545186c85befbf8 2024-11-13T16:08:13,337 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for bf1f9f9bc1bc9ffce566d41f8ae3619a 2024-11-13T16:08:13,337 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for bf1f9f9bc1bc9ffce566d41f8ae3619a 2024-11-13T16:08:13,337 INFO [StoreOpener-29df624bbcb878edce7fcdf0abceea88-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 29df624bbcb878edce7fcdf0abceea88 2024-11-13T16:08:13,338 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T16:08:13,338 INFO [StoreOpener-7c34e9ded13254b91545186c85befbf8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 7c34e9ded13254b91545186c85befbf8 2024-11-13T16:08:13,340 INFO [StoreOpener-7c34e9ded13254b91545186c85befbf8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7c34e9ded13254b91545186c85befbf8 columnFamilyName colfamily11 2024-11-13T16:08:13,340 INFO [StoreOpener-29df624bbcb878edce7fcdf0abceea88-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 29df624bbcb878edce7fcdf0abceea88 columnFamilyName colfamily11 2024-11-13T16:08:13,340 DEBUG [StoreOpener-7c34e9ded13254b91545186c85befbf8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,340 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for bf1f9f9bc1bc9ffce566d41f8ae3619a 2024-11-13T16:08:13,340 DEBUG [StoreOpener-29df624bbcb878edce7fcdf0abceea88-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,341 INFO [StoreOpener-7c34e9ded13254b91545186c85befbf8-1 {}] regionserver.HStore(327): Store=7c34e9ded13254b91545186c85befbf8/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,341 INFO [StoreOpener-29df624bbcb878edce7fcdf0abceea88-1 {}] regionserver.HStore(327): Store=29df624bbcb878edce7fcdf0abceea88/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,341 INFO [StoreOpener-7c34e9ded13254b91545186c85befbf8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 7c34e9ded13254b91545186c85befbf8 2024-11-13T16:08:13,341 INFO [StoreOpener-29df624bbcb878edce7fcdf0abceea88-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 29df624bbcb878edce7fcdf0abceea88 2024-11-13T16:08:13,343 INFO [StoreOpener-29df624bbcb878edce7fcdf0abceea88-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 29df624bbcb878edce7fcdf0abceea88 columnFamilyName colfamily21 2024-11-13T16:08:13,343 INFO [StoreOpener-7c34e9ded13254b91545186c85befbf8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7c34e9ded13254b91545186c85befbf8 columnFamilyName colfamily21 2024-11-13T16:08:13,343 DEBUG [StoreOpener-29df624bbcb878edce7fcdf0abceea88-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,343 DEBUG [StoreOpener-7c34e9ded13254b91545186c85befbf8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,344 INFO [StoreOpener-29df624bbcb878edce7fcdf0abceea88-1 {}] regionserver.HStore(327): Store=29df624bbcb878edce7fcdf0abceea88/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,344 INFO [StoreOpener-7c34e9ded13254b91545186c85befbf8-1 {}] regionserver.HStore(327): Store=7c34e9ded13254b91545186c85befbf8/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,344 INFO [StoreOpener-29df624bbcb878edce7fcdf0abceea88-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 29df624bbcb878edce7fcdf0abceea88 2024-11-13T16:08:13,344 INFO [StoreOpener-7c34e9ded13254b91545186c85befbf8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 7c34e9ded13254b91545186c85befbf8 2024-11-13T16:08:13,345 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bf1f9f9bc1bc9ffce566d41f8ae3619a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T16:08:13,346 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened bf1f9f9bc1bc9ffce566d41f8ae3619a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70280717, jitterRate=0.04726429283618927}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T16:08:13,346 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bf1f9f9bc1bc9ffce566d41f8ae3619a 2024-11-13T16:08:13,346 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for bf1f9f9bc1bc9ffce566d41f8ae3619a: Running coprocessor pre-open hook at 1731514093306Writing region info on filesystem at 1731514093306Initializing all the Stores at 1731514093307 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093307Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093317 (+10 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093317Cleaning up temporary data from old regions at 1731514093337 (+20 ms)Running coprocessor post-open hooks at 1731514093346 (+9 ms)Region opened successfully at 1731514093346 2024-11-13T16:08:13,347 INFO [StoreOpener-7c34e9ded13254b91545186c85befbf8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7c34e9ded13254b91545186c85befbf8 columnFamilyName colfamily31 2024-11-13T16:08:13,347 INFO [StoreOpener-29df624bbcb878edce7fcdf0abceea88-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 29df624bbcb878edce7fcdf0abceea88 columnFamilyName colfamily31 2024-11-13T16:08:13,347 DEBUG [StoreOpener-7c34e9ded13254b91545186c85befbf8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,347 DEBUG [StoreOpener-29df624bbcb878edce7fcdf0abceea88-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,347 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=3647edbd8aef23dda3887b61277da1d3, regionState=OPEN, openSeqNum=2, regionLocation=f8f85b0bb1d8,38187,1731514088725 2024-11-13T16:08:13,348 INFO [StoreOpener-7c34e9ded13254b91545186c85befbf8-1 {}] regionserver.HStore(327): Store=7c34e9ded13254b91545186c85befbf8/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,348 INFO [StoreOpener-29df624bbcb878edce7fcdf0abceea88-1 {}] regionserver.HStore(327): Store=29df624bbcb878edce7fcdf0abceea88/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,348 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for 7c34e9ded13254b91545186c85befbf8 2024-11-13T16:08:13,348 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 29df624bbcb878edce7fcdf0abceea88 2024-11-13T16:08:13,349 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731514091776.bf1f9f9bc1bc9ffce566d41f8ae3619a., pid=33, masterSystemTime=1731514093089 2024-11-13T16:08:13,349 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7c34e9ded13254b91545186c85befbf8 2024-11-13T16:08:13,350 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/29df624bbcb878edce7fcdf0abceea88 2024-11-13T16:08:13,350 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7c34e9ded13254b91545186c85befbf8 2024-11-13T16:08:13,351 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/29df624bbcb878edce7fcdf0abceea88 2024-11-13T16:08:13,352 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3647edbd8aef23dda3887b61277da1d3, server=f8f85b0bb1d8,38187,1731514088725 because future has completed 2024-11-13T16:08:13,352 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 29df624bbcb878edce7fcdf0abceea88 2024-11-13T16:08:13,353 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for 7c34e9ded13254b91545186c85befbf8 2024-11-13T16:08:13,353 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 29df624bbcb878edce7fcdf0abceea88 2024-11-13T16:08:13,353 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for 7c34e9ded13254b91545186c85befbf8 2024-11-13T16:08:13,353 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T16:08:13,353 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T16:08:13,355 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731514091776.bf1f9f9bc1bc9ffce566d41f8ae3619a. 2024-11-13T16:08:13,355 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731514091776.bf1f9f9bc1bc9ffce566d41f8ae3619a. 2024-11-13T16:08:13,356 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731514091776.4aaf5667f132c289386d1b3b0c0ac57f. 2024-11-13T16:08:13,356 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => 4aaf5667f132c289386d1b3b0c0ac57f, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731514091776.4aaf5667f132c289386d1b3b0c0ac57f.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-13T16:08:13,356 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for 7c34e9ded13254b91545186c85befbf8 2024-11-13T16:08:13,357 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 4aaf5667f132c289386d1b3b0c0ac57f 2024-11-13T16:08:13,357 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731514091776.4aaf5667f132c289386d1b3b0c0ac57f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:13,357 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for 4aaf5667f132c289386d1b3b0c0ac57f 2024-11-13T16:08:13,357 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 29df624bbcb878edce7fcdf0abceea88 2024-11-13T16:08:13,357 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for 4aaf5667f132c289386d1b3b0c0ac57f 2024-11-13T16:08:13,359 INFO [StoreOpener-4aaf5667f132c289386d1b3b0c0ac57f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 4aaf5667f132c289386d1b3b0c0ac57f 2024-11-13T16:08:13,364 INFO [StoreOpener-4aaf5667f132c289386d1b3b0c0ac57f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4aaf5667f132c289386d1b3b0c0ac57f columnFamilyName colfamily11 2024-11-13T16:08:13,364 DEBUG [StoreOpener-4aaf5667f132c289386d1b3b0c0ac57f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,365 INFO [StoreOpener-4aaf5667f132c289386d1b3b0c0ac57f-1 {}] regionserver.HStore(327): Store=4aaf5667f132c289386d1b3b0c0ac57f/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,366 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7c34e9ded13254b91545186c85befbf8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T16:08:13,366 INFO [StoreOpener-4aaf5667f132c289386d1b3b0c0ac57f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 4aaf5667f132c289386d1b3b0c0ac57f 2024-11-13T16:08:13,366 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=18 2024-11-13T16:08:13,366 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure 3647edbd8aef23dda3887b61277da1d3, server=f8f85b0bb1d8,38187,1731514088725 in 196 msec 2024-11-13T16:08:13,367 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened 7c34e9ded13254b91545186c85befbf8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72239636, jitterRate=0.07645446062088013}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T16:08:13,368 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7c34e9ded13254b91545186c85befbf8 2024-11-13T16:08:13,368 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for 7c34e9ded13254b91545186c85befbf8: Running coprocessor pre-open hook at 1731514093336Writing region info on filesystem at 1731514093336Initializing all the Stores at 1731514093337 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093337Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093338 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093338Cleaning up temporary data from old regions at 1731514093353 (+15 ms)Running coprocessor post-open hooks at 1731514093368 (+15 ms)Region opened successfully at 1731514093368 2024-11-13T16:08:13,369 INFO [StoreOpener-4aaf5667f132c289386d1b3b0c0ac57f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4aaf5667f132c289386d1b3b0c0ac57f columnFamilyName colfamily21 2024-11-13T16:08:13,369 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731514091776.7c34e9ded13254b91545186c85befbf8., pid=51, masterSystemTime=1731514093227 2024-11-13T16:08:13,369 DEBUG [StoreOpener-4aaf5667f132c289386d1b3b0c0ac57f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,371 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/29df624bbcb878edce7fcdf0abceea88/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T16:08:13,372 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 29df624bbcb878edce7fcdf0abceea88; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59363142, jitterRate=-0.11542025208473206}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T16:08:13,372 INFO [StoreOpener-4aaf5667f132c289386d1b3b0c0ac57f-1 {}] regionserver.HStore(327): Store=4aaf5667f132c289386d1b3b0c0ac57f/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,372 INFO [StoreOpener-4aaf5667f132c289386d1b3b0c0ac57f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 4aaf5667f132c289386d1b3b0c0ac57f 2024-11-13T16:08:13,372 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 29df624bbcb878edce7fcdf0abceea88 2024-11-13T16:08:13,373 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 29df624bbcb878edce7fcdf0abceea88: Running coprocessor pre-open hook at 1731514093335Writing region info on filesystem at 1731514093335Initializing all the Stores at 1731514093337 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093337Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093337Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093337Cleaning up temporary data from old regions at 1731514093353 (+16 ms)Running coprocessor post-open hooks at 1731514093373 (+20 ms)Region opened successfully at 1731514093373 2024-11-13T16:08:13,375 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731514091776.29df624bbcb878edce7fcdf0abceea88., pid=39, masterSystemTime=1731514093033 2024-11-13T16:08:13,376 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3647edbd8aef23dda3887b61277da1d3, ASSIGN in 683 msec 2024-11-13T16:08:13,376 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731514091776.7c34e9ded13254b91545186c85befbf8. 2024-11-13T16:08:13,377 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731514091776.7c34e9ded13254b91545186c85befbf8. 2024-11-13T16:08:13,377 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731514091776.4a146f98682f25e8cddcae5dbf318603. 2024-11-13T16:08:13,377 INFO [StoreOpener-4aaf5667f132c289386d1b3b0c0ac57f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4aaf5667f132c289386d1b3b0c0ac57f columnFamilyName colfamily31 2024-11-13T16:08:13,377 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => 4a146f98682f25e8cddcae5dbf318603, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731514091776.4a146f98682f25e8cddcae5dbf318603.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-13T16:08:13,377 DEBUG [StoreOpener-4aaf5667f132c289386d1b3b0c0ac57f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,377 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 4a146f98682f25e8cddcae5dbf318603 2024-11-13T16:08:13,377 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731514091776.4a146f98682f25e8cddcae5dbf318603.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:13,378 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for 4a146f98682f25e8cddcae5dbf318603 2024-11-13T16:08:13,378 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for 4a146f98682f25e8cddcae5dbf318603 2024-11-13T16:08:13,378 INFO [StoreOpener-4aaf5667f132c289386d1b3b0c0ac57f-1 {}] regionserver.HStore(327): Store=4aaf5667f132c289386d1b3b0c0ac57f/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,379 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for 4aaf5667f132c289386d1b3b0c0ac57f 2024-11-13T16:08:13,380 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4aaf5667f132c289386d1b3b0c0ac57f 2024-11-13T16:08:13,380 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=7c34e9ded13254b91545186c85befbf8, regionState=OPEN, openSeqNum=2, regionLocation=f8f85b0bb1d8,38187,1731514088725 2024-11-13T16:08:13,381 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4aaf5667f132c289386d1b3b0c0ac57f 2024-11-13T16:08:13,384 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for 4aaf5667f132c289386d1b3b0c0ac57f 2024-11-13T16:08:13,384 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for 4aaf5667f132c289386d1b3b0c0ac57f 2024-11-13T16:08:13,385 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T16:08:13,387 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731514091776.29df624bbcb878edce7fcdf0abceea88. 2024-11-13T16:08:13,387 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731514091776.29df624bbcb878edce7fcdf0abceea88. 2024-11-13T16:08:13,387 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731514091776.fec5f21b48c36bf01685ec2ee58f0490. 2024-11-13T16:08:13,387 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => fec5f21b48c36bf01685ec2ee58f0490, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731514091776.fec5f21b48c36bf01685ec2ee58f0490.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-13T16:08:13,387 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=29df624bbcb878edce7fcdf0abceea88, regionState=OPEN, openSeqNum=2, regionLocation=f8f85b0bb1d8,43467,1731514088560 2024-11-13T16:08:13,387 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true fec5f21b48c36bf01685ec2ee58f0490 2024-11-13T16:08:13,388 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731514091776.fec5f21b48c36bf01685ec2ee58f0490.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:13,388 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for fec5f21b48c36bf01685ec2ee58f0490 2024-11-13T16:08:13,388 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for fec5f21b48c36bf01685ec2ee58f0490 2024-11-13T16:08:13,388 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7c34e9ded13254b91545186c85befbf8, server=f8f85b0bb1d8,38187,1731514088725 because future has completed 2024-11-13T16:08:13,389 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for 4aaf5667f132c289386d1b3b0c0ac57f 2024-11-13T16:08:13,391 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 29df624bbcb878edce7fcdf0abceea88, server=f8f85b0bb1d8,43467,1731514088560 because future has completed 2024-11-13T16:08:13,395 INFO [StoreOpener-4a146f98682f25e8cddcae5dbf318603-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 4a146f98682f25e8cddcae5dbf318603 2024-11-13T16:08:13,397 INFO [StoreOpener-4a146f98682f25e8cddcae5dbf318603-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4a146f98682f25e8cddcae5dbf318603 columnFamilyName colfamily11 2024-11-13T16:08:13,398 DEBUG [StoreOpener-4a146f98682f25e8cddcae5dbf318603-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,402 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=21 2024-11-13T16:08:13,402 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure 7c34e9ded13254b91545186c85befbf8, server=f8f85b0bb1d8,38187,1731514088725 in 213 msec 2024-11-13T16:08:13,405 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7c34e9ded13254b91545186c85befbf8, ASSIGN in 718 msec 2024-11-13T16:08:13,407 INFO [StoreOpener-fec5f21b48c36bf01685ec2ee58f0490-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region fec5f21b48c36bf01685ec2ee58f0490 2024-11-13T16:08:13,408 INFO [StoreOpener-4a146f98682f25e8cddcae5dbf318603-1 {}] regionserver.HStore(327): Store=4a146f98682f25e8cddcae5dbf318603/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,408 INFO [StoreOpener-4a146f98682f25e8cddcae5dbf318603-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 4a146f98682f25e8cddcae5dbf318603 2024-11-13T16:08:13,409 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4aaf5667f132c289386d1b3b0c0ac57f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T16:08:13,410 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened 4aaf5667f132c289386d1b3b0c0ac57f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70636506, jitterRate=0.052565962076187134}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T16:08:13,410 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4aaf5667f132c289386d1b3b0c0ac57f 2024-11-13T16:08:13,411 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for 4aaf5667f132c289386d1b3b0c0ac57f: Running coprocessor pre-open hook at 1731514093357Writing region info on filesystem at 1731514093357Initializing all the Stores at 1731514093359 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093359Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093359Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093359Cleaning up temporary data from old regions at 1731514093384 (+25 ms)Running coprocessor post-open hooks at 1731514093410 (+26 ms)Region opened successfully at 1731514093410 2024-11-13T16:08:13,411 INFO [StoreOpener-fec5f21b48c36bf01685ec2ee58f0490-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fec5f21b48c36bf01685ec2ee58f0490 columnFamilyName colfamily11 2024-11-13T16:08:13,411 DEBUG [StoreOpener-fec5f21b48c36bf01685ec2ee58f0490-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,412 INFO [StoreOpener-fec5f21b48c36bf01685ec2ee58f0490-1 {}] regionserver.HStore(327): Store=fec5f21b48c36bf01685ec2ee58f0490/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,412 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42949 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=f8f85b0bb1d8,41333,1731514088798, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e72e57b96ff9bc0bbd862ce564d79c42. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T16:08:13,412 INFO [StoreOpener-fec5f21b48c36bf01685ec2ee58f0490-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region fec5f21b48c36bf01685ec2ee58f0490 2024-11-13T16:08:13,412 INFO [StoreOpener-4a146f98682f25e8cddcae5dbf318603-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4a146f98682f25e8cddcae5dbf318603 columnFamilyName colfamily21 2024-11-13T16:08:13,412 DEBUG [StoreOpener-4a146f98682f25e8cddcae5dbf318603-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,413 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731514091776.4aaf5667f132c289386d1b3b0c0ac57f., pid=44, masterSystemTime=1731514093089 2024-11-13T16:08:13,415 INFO [StoreOpener-4a146f98682f25e8cddcae5dbf318603-1 {}] regionserver.HStore(327): Store=4a146f98682f25e8cddcae5dbf318603/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,415 INFO [StoreOpener-4a146f98682f25e8cddcae5dbf318603-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 4a146f98682f25e8cddcae5dbf318603 2024-11-13T16:08:13,415 INFO [StoreOpener-fec5f21b48c36bf01685ec2ee58f0490-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fec5f21b48c36bf01685ec2ee58f0490 columnFamilyName colfamily21 2024-11-13T16:08:13,415 DEBUG [StoreOpener-fec5f21b48c36bf01685ec2ee58f0490-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,416 INFO [StoreOpener-fec5f21b48c36bf01685ec2ee58f0490-1 {}] regionserver.HStore(327): Store=fec5f21b48c36bf01685ec2ee58f0490/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,417 INFO [StoreOpener-fec5f21b48c36bf01685ec2ee58f0490-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region fec5f21b48c36bf01685ec2ee58f0490 2024-11-13T16:08:13,418 INFO [StoreOpener-4a146f98682f25e8cddcae5dbf318603-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4a146f98682f25e8cddcae5dbf318603 columnFamilyName colfamily31 2024-11-13T16:08:13,418 DEBUG [StoreOpener-4a146f98682f25e8cddcae5dbf318603-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,419 INFO [StoreOpener-4a146f98682f25e8cddcae5dbf318603-1 {}] regionserver.HStore(327): Store=4a146f98682f25e8cddcae5dbf318603/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,419 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731514091776.4aaf5667f132c289386d1b3b0c0ac57f. 2024-11-13T16:08:13,419 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for 4a146f98682f25e8cddcae5dbf318603 2024-11-13T16:08:13,419 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731514091776.4aaf5667f132c289386d1b3b0c0ac57f. 2024-11-13T16:08:13,419 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731514091776.fe97fe0e9b37811ba99ca8100b461589. 2024-11-13T16:08:13,419 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => fe97fe0e9b37811ba99ca8100b461589, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731514091776.fe97fe0e9b37811ba99ca8100b461589.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-13T16:08:13,420 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=19 2024-11-13T16:08:13,420 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true fe97fe0e9b37811ba99ca8100b461589 2024-11-13T16:08:13,420 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure 29df624bbcb878edce7fcdf0abceea88, server=f8f85b0bb1d8,43467,1731514088560 in 423 msec 2024-11-13T16:08:13,420 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731514091776.fe97fe0e9b37811ba99ca8100b461589.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:13,420 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for fe97fe0e9b37811ba99ca8100b461589 2024-11-13T16:08:13,420 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for fe97fe0e9b37811ba99ca8100b461589 2024-11-13T16:08:13,420 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4a146f98682f25e8cddcae5dbf318603 2024-11-13T16:08:13,422 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4a146f98682f25e8cddcae5dbf318603 2024-11-13T16:08:13,422 INFO [StoreOpener-fec5f21b48c36bf01685ec2ee58f0490-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fec5f21b48c36bf01685ec2ee58f0490 columnFamilyName colfamily31 2024-11-13T16:08:13,422 DEBUG [StoreOpener-fec5f21b48c36bf01685ec2ee58f0490-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,424 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for 4a146f98682f25e8cddcae5dbf318603 2024-11-13T16:08:13,424 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for 4a146f98682f25e8cddcae5dbf318603 2024-11-13T16:08:13,425 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T16:08:13,425 INFO [StoreOpener-fec5f21b48c36bf01685ec2ee58f0490-1 {}] regionserver.HStore(327): Store=fec5f21b48c36bf01685ec2ee58f0490/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,425 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for fec5f21b48c36bf01685ec2ee58f0490 2024-11-13T16:08:13,427 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fec5f21b48c36bf01685ec2ee58f0490 2024-11-13T16:08:13,427 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fec5f21b48c36bf01685ec2ee58f0490 2024-11-13T16:08:13,428 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=29df624bbcb878edce7fcdf0abceea88, ASSIGN in 736 msec 2024-11-13T16:08:13,429 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for 4a146f98682f25e8cddcae5dbf318603 2024-11-13T16:08:13,429 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for fec5f21b48c36bf01685ec2ee58f0490 2024-11-13T16:08:13,429 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for fec5f21b48c36bf01685ec2ee58f0490 2024-11-13T16:08:13,430 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4aaf5667f132c289386d1b3b0c0ac57f, regionState=OPEN, openSeqNum=2, regionLocation=f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:08:13,431 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T16:08:13,434 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for fec5f21b48c36bf01685ec2ee58f0490 2024-11-13T16:08:13,435 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4aaf5667f132c289386d1b3b0c0ac57f, server=f8f85b0bb1d8,41333,1731514088798 because future has completed 2024-11-13T16:08:13,436 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=bf1f9f9bc1bc9ffce566d41f8ae3619a, regionState=OPEN, openSeqNum=2, regionLocation=f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:08:13,441 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure bf1f9f9bc1bc9ffce566d41f8ae3619a, server=f8f85b0bb1d8,41333,1731514088798 because future has completed 2024-11-13T16:08:13,443 INFO [StoreOpener-fe97fe0e9b37811ba99ca8100b461589-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region fe97fe0e9b37811ba99ca8100b461589 2024-11-13T16:08:13,446 INFO [StoreOpener-fe97fe0e9b37811ba99ca8100b461589-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fe97fe0e9b37811ba99ca8100b461589 columnFamilyName colfamily11 2024-11-13T16:08:13,446 DEBUG [StoreOpener-fe97fe0e9b37811ba99ca8100b461589-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,447 INFO [StoreOpener-fe97fe0e9b37811ba99ca8100b461589-1 {}] regionserver.HStore(327): Store=fe97fe0e9b37811ba99ca8100b461589/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,447 INFO [StoreOpener-fe97fe0e9b37811ba99ca8100b461589-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region fe97fe0e9b37811ba99ca8100b461589 2024-11-13T16:08:13,447 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fec5f21b48c36bf01685ec2ee58f0490/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T16:08:13,448 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened fec5f21b48c36bf01685ec2ee58f0490; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71059351, jitterRate=0.058866843581199646}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T16:08:13,448 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fec5f21b48c36bf01685ec2ee58f0490 2024-11-13T16:08:13,448 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for fec5f21b48c36bf01685ec2ee58f0490: Running coprocessor pre-open hook at 1731514093388Writing region info on filesystem at 1731514093388Initializing all the Stores at 1731514093390 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093390Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093407 (+17 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093407Cleaning up temporary data from old regions at 1731514093429 (+22 ms)Running coprocessor post-open hooks at 1731514093448 (+19 ms)Region opened successfully at 1731514093448 2024-11-13T16:08:13,450 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=5 2024-11-13T16:08:13,450 INFO [StoreOpener-fe97fe0e9b37811ba99ca8100b461589-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fe97fe0e9b37811ba99ca8100b461589 columnFamilyName colfamily21 2024-11-13T16:08:13,450 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 4aaf5667f132c289386d1b3b0c0ac57f, server=f8f85b0bb1d8,41333,1731514088798 in 399 msec 2024-11-13T16:08:13,451 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731514091776.fec5f21b48c36bf01685ec2ee58f0490., pid=40, masterSystemTime=1731514093033 2024-11-13T16:08:13,451 DEBUG [StoreOpener-fe97fe0e9b37811ba99ca8100b461589-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,453 INFO [StoreOpener-fe97fe0e9b37811ba99ca8100b461589-1 {}] regionserver.HStore(327): Store=fe97fe0e9b37811ba99ca8100b461589/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,453 INFO [StoreOpener-fe97fe0e9b37811ba99ca8100b461589-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region fe97fe0e9b37811ba99ca8100b461589 2024-11-13T16:08:13,456 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731514091776.fec5f21b48c36bf01685ec2ee58f0490. 2024-11-13T16:08:13,456 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731514091776.fec5f21b48c36bf01685ec2ee58f0490. 2024-11-13T16:08:13,456 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731514091776.519ece296a21b5ddccc38a7c928780bb. 2024-11-13T16:08:13,457 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 519ece296a21b5ddccc38a7c928780bb, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731514091776.519ece296a21b5ddccc38a7c928780bb.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-13T16:08:13,457 INFO [StoreOpener-fe97fe0e9b37811ba99ca8100b461589-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fe97fe0e9b37811ba99ca8100b461589 columnFamilyName colfamily31 2024-11-13T16:08:13,457 DEBUG [StoreOpener-fe97fe0e9b37811ba99ca8100b461589-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,457 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 519ece296a21b5ddccc38a7c928780bb 2024-11-13T16:08:13,457 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731514091776.519ece296a21b5ddccc38a7c928780bb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:13,457 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 519ece296a21b5ddccc38a7c928780bb 2024-11-13T16:08:13,457 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 519ece296a21b5ddccc38a7c928780bb 2024-11-13T16:08:13,459 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4aaf5667f132c289386d1b3b0c0ac57f, ASSIGN in 767 msec 2024-11-13T16:08:13,460 INFO [StoreOpener-fe97fe0e9b37811ba99ca8100b461589-1 {}] regionserver.HStore(327): Store=fe97fe0e9b37811ba99ca8100b461589/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,460 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for fe97fe0e9b37811ba99ca8100b461589 2024-11-13T16:08:13,463 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fe97fe0e9b37811ba99ca8100b461589 2024-11-13T16:08:13,464 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=fec5f21b48c36bf01685ec2ee58f0490, regionState=OPEN, openSeqNum=2, regionLocation=f8f85b0bb1d8,43467,1731514088560 2024-11-13T16:08:13,464 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4a146f98682f25e8cddcae5dbf318603/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T16:08:13,464 INFO [StoreOpener-519ece296a21b5ddccc38a7c928780bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 519ece296a21b5ddccc38a7c928780bb 2024-11-13T16:08:13,464 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fe97fe0e9b37811ba99ca8100b461589 2024-11-13T16:08:13,465 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened 4a146f98682f25e8cddcae5dbf318603; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67213313, jitterRate=0.0015564113855361938}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T16:08:13,465 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4a146f98682f25e8cddcae5dbf318603 2024-11-13T16:08:13,465 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for 4a146f98682f25e8cddcae5dbf318603: Running coprocessor pre-open hook at 1731514093378Writing region info on filesystem at 1731514093378Initializing all the Stores at 1731514093380 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093380Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093394 (+14 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093395 (+1 ms)Cleaning up temporary data from old regions at 1731514093424 (+29 ms)Running coprocessor post-open hooks at 1731514093465 (+41 ms)Region opened successfully at 1731514093465 2024-11-13T16:08:13,466 INFO [StoreOpener-519ece296a21b5ddccc38a7c928780bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 519ece296a21b5ddccc38a7c928780bb columnFamilyName colfamily11 2024-11-13T16:08:13,466 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731514091776.4a146f98682f25e8cddcae5dbf318603., pid=45, masterSystemTime=1731514093227 2024-11-13T16:08:13,467 DEBUG [StoreOpener-519ece296a21b5ddccc38a7c928780bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,468 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure fec5f21b48c36bf01685ec2ee58f0490, server=f8f85b0bb1d8,43467,1731514088560 because future has completed 2024-11-13T16:08:13,468 INFO [StoreOpener-519ece296a21b5ddccc38a7c928780bb-1 {}] regionserver.HStore(327): Store=519ece296a21b5ddccc38a7c928780bb/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,468 INFO [StoreOpener-519ece296a21b5ddccc38a7c928780bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 519ece296a21b5ddccc38a7c928780bb 2024-11-13T16:08:13,469 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for fe97fe0e9b37811ba99ca8100b461589 2024-11-13T16:08:13,469 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for fe97fe0e9b37811ba99ca8100b461589 2024-11-13T16:08:13,472 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T16:08:13,472 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731514091776.4a146f98682f25e8cddcae5dbf318603. 2024-11-13T16:08:13,472 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731514091776.4a146f98682f25e8cddcae5dbf318603. 2024-11-13T16:08:13,472 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731514091776.c4a87adf770b37d40ab6691e377ad1b5. 2024-11-13T16:08:13,472 INFO [StoreOpener-519ece296a21b5ddccc38a7c928780bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 519ece296a21b5ddccc38a7c928780bb columnFamilyName colfamily21 2024-11-13T16:08:13,473 DEBUG [StoreOpener-519ece296a21b5ddccc38a7c928780bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,473 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => c4a87adf770b37d40ab6691e377ad1b5, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731514091776.c4a87adf770b37d40ab6691e377ad1b5.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-13T16:08:13,475 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for fe97fe0e9b37811ba99ca8100b461589 2024-11-13T16:08:13,475 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true c4a87adf770b37d40ab6691e377ad1b5 2024-11-13T16:08:13,482 INFO [StoreOpener-519ece296a21b5ddccc38a7c928780bb-1 {}] regionserver.HStore(327): Store=519ece296a21b5ddccc38a7c928780bb/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,485 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731514091776.c4a87adf770b37d40ab6691e377ad1b5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:13,485 INFO [StoreOpener-519ece296a21b5ddccc38a7c928780bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 519ece296a21b5ddccc38a7c928780bb 2024-11-13T16:08:13,485 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for c4a87adf770b37d40ab6691e377ad1b5 2024-11-13T16:08:13,485 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for c4a87adf770b37d40ab6691e377ad1b5 2024-11-13T16:08:13,487 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42949 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=f8f85b0bb1d8,43467,1731514088560, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fec5f21b48c36bf01685ec2ee58f0490. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T16:08:13,488 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fe97fe0e9b37811ba99ca8100b461589/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T16:08:13,489 INFO [StoreOpener-519ece296a21b5ddccc38a7c928780bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 519ece296a21b5ddccc38a7c928780bb columnFamilyName colfamily31 2024-11-13T16:08:13,489 DEBUG [StoreOpener-519ece296a21b5ddccc38a7c928780bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,490 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened fe97fe0e9b37811ba99ca8100b461589; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63035410, jitterRate=-0.06069919466972351}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T16:08:13,491 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fe97fe0e9b37811ba99ca8100b461589 2024-11-13T16:08:13,491 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for fe97fe0e9b37811ba99ca8100b461589: Running coprocessor pre-open hook at 1731514093421Writing region info on filesystem at 1731514093421Initializing all the Stores at 1731514093423 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093423Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093442 (+19 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093443 (+1 ms)Cleaning up temporary data from old regions at 1731514093469 (+26 ms)Running coprocessor post-open hooks at 1731514093491 (+22 ms)Region opened successfully at 1731514093491 2024-11-13T16:08:13,492 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=27 2024-11-13T16:08:13,492 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure bf1f9f9bc1bc9ffce566d41f8ae3619a, server=f8f85b0bb1d8,41333,1731514088798 in 542 msec 2024-11-13T16:08:13,493 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731514091776.fe97fe0e9b37811ba99ca8100b461589., pid=42, masterSystemTime=1731514093089 2024-11-13T16:08:13,494 INFO [StoreOpener-519ece296a21b5ddccc38a7c928780bb-1 {}] regionserver.HStore(327): Store=519ece296a21b5ddccc38a7c928780bb/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,494 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 519ece296a21b5ddccc38a7c928780bb 2024-11-13T16:08:13,496 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/519ece296a21b5ddccc38a7c928780bb 2024-11-13T16:08:13,498 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/519ece296a21b5ddccc38a7c928780bb 2024-11-13T16:08:13,501 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 519ece296a21b5ddccc38a7c928780bb 2024-11-13T16:08:13,501 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 519ece296a21b5ddccc38a7c928780bb 2024-11-13T16:08:13,501 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731514091776.fe97fe0e9b37811ba99ca8100b461589. 2024-11-13T16:08:13,501 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731514091776.fe97fe0e9b37811ba99ca8100b461589. 2024-11-13T16:08:13,501 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731514091776.72d273770314688e78624993de4f3398. 2024-11-13T16:08:13,502 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 72d273770314688e78624993de4f3398, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731514091776.72d273770314688e78624993de4f3398.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-13T16:08:13,502 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T16:08:13,502 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=bf1f9f9bc1bc9ffce566d41f8ae3619a, ASSIGN in 808 msec 2024-11-13T16:08:13,502 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 72d273770314688e78624993de4f3398 2024-11-13T16:08:13,502 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731514091776.72d273770314688e78624993de4f3398.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:13,503 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 72d273770314688e78624993de4f3398 2024-11-13T16:08:13,503 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 72d273770314688e78624993de4f3398 2024-11-13T16:08:13,505 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 519ece296a21b5ddccc38a7c928780bb 2024-11-13T16:08:13,505 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=fe97fe0e9b37811ba99ca8100b461589, regionState=OPEN, openSeqNum=2, regionLocation=f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:08:13,507 INFO [StoreOpener-c4a87adf770b37d40ab6691e377ad1b5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region c4a87adf770b37d40ab6691e377ad1b5 2024-11-13T16:08:13,512 INFO [StoreOpener-c4a87adf770b37d40ab6691e377ad1b5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c4a87adf770b37d40ab6691e377ad1b5 columnFamilyName colfamily11 2024-11-13T16:08:13,512 DEBUG [StoreOpener-c4a87adf770b37d40ab6691e377ad1b5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,512 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure fe97fe0e9b37811ba99ca8100b461589, server=f8f85b0bb1d8,41333,1731514088798 because future has completed 2024-11-13T16:08:13,513 INFO [StoreOpener-c4a87adf770b37d40ab6691e377ad1b5-1 {}] regionserver.HStore(327): Store=c4a87adf770b37d40ab6691e377ad1b5/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,513 INFO [StoreOpener-c4a87adf770b37d40ab6691e377ad1b5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region c4a87adf770b37d40ab6691e377ad1b5 2024-11-13T16:08:13,516 INFO [StoreOpener-72d273770314688e78624993de4f3398-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 72d273770314688e78624993de4f3398 2024-11-13T16:08:13,519 INFO [StoreOpener-c4a87adf770b37d40ab6691e377ad1b5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c4a87adf770b37d40ab6691e377ad1b5 columnFamilyName colfamily21 2024-11-13T16:08:13,519 INFO [StoreOpener-72d273770314688e78624993de4f3398-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 72d273770314688e78624993de4f3398 columnFamilyName colfamily11 2024-11-13T16:08:13,520 DEBUG [StoreOpener-72d273770314688e78624993de4f3398-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,520 DEBUG [StoreOpener-c4a87adf770b37d40ab6691e377ad1b5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,520 INFO [StoreOpener-72d273770314688e78624993de4f3398-1 {}] regionserver.HStore(327): Store=72d273770314688e78624993de4f3398/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,520 INFO [StoreOpener-c4a87adf770b37d40ab6691e377ad1b5-1 {}] regionserver.HStore(327): Store=c4a87adf770b37d40ab6691e377ad1b5/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,521 INFO [StoreOpener-72d273770314688e78624993de4f3398-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 72d273770314688e78624993de4f3398 2024-11-13T16:08:13,521 INFO [StoreOpener-c4a87adf770b37d40ab6691e377ad1b5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region c4a87adf770b37d40ab6691e377ad1b5 2024-11-13T16:08:13,523 INFO [StoreOpener-c4a87adf770b37d40ab6691e377ad1b5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c4a87adf770b37d40ab6691e377ad1b5 columnFamilyName colfamily31 2024-11-13T16:08:13,523 DEBUG [StoreOpener-c4a87adf770b37d40ab6691e377ad1b5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,524 INFO [StoreOpener-c4a87adf770b37d40ab6691e377ad1b5-1 {}] regionserver.HStore(327): Store=c4a87adf770b37d40ab6691e377ad1b5/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,525 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for c4a87adf770b37d40ab6691e377ad1b5 2024-11-13T16:08:13,525 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=4a146f98682f25e8cddcae5dbf318603, regionState=OPEN, openSeqNum=2, regionLocation=f8f85b0bb1d8,38187,1731514088725 2024-11-13T16:08:13,526 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c4a87adf770b37d40ab6691e377ad1b5 2024-11-13T16:08:13,527 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c4a87adf770b37d40ab6691e377ad1b5 2024-11-13T16:08:13,530 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for c4a87adf770b37d40ab6691e377ad1b5 2024-11-13T16:08:13,530 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for c4a87adf770b37d40ab6691e377ad1b5 2024-11-13T16:08:13,531 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/519ece296a21b5ddccc38a7c928780bb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T16:08:13,531 INFO [StoreOpener-72d273770314688e78624993de4f3398-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 72d273770314688e78624993de4f3398 columnFamilyName colfamily21 2024-11-13T16:08:13,532 DEBUG [StoreOpener-72d273770314688e78624993de4f3398-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,532 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T16:08:13,533 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4a146f98682f25e8cddcae5dbf318603, server=f8f85b0bb1d8,38187,1731514088725 because future has completed 2024-11-13T16:08:13,533 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 519ece296a21b5ddccc38a7c928780bb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59200957, jitterRate=-0.11783699691295624}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T16:08:13,534 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 519ece296a21b5ddccc38a7c928780bb 2024-11-13T16:08:13,534 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 519ece296a21b5ddccc38a7c928780bb: Running coprocessor pre-open hook at 1731514093458Writing region info on filesystem at 1731514093458Initializing all the Stores at 1731514093463 (+5 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093463Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093463Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093463Cleaning up temporary data from old regions at 1731514093501 (+38 ms)Running coprocessor post-open hooks at 1731514093534 (+33 ms)Region opened successfully at 1731514093534 2024-11-13T16:08:13,536 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731514091776.519ece296a21b5ddccc38a7c928780bb., pid=41, masterSystemTime=1731514093033 2024-11-13T16:08:13,537 INFO [StoreOpener-72d273770314688e78624993de4f3398-1 {}] regionserver.HStore(327): Store=72d273770314688e78624993de4f3398/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,537 INFO [StoreOpener-72d273770314688e78624993de4f3398-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 72d273770314688e78624993de4f3398 2024-11-13T16:08:13,542 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for c4a87adf770b37d40ab6691e377ad1b5 2024-11-13T16:08:13,547 INFO [StoreOpener-72d273770314688e78624993de4f3398-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 72d273770314688e78624993de4f3398 columnFamilyName colfamily31 2024-11-13T16:08:13,547 DEBUG [StoreOpener-72d273770314688e78624993de4f3398-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,551 INFO [StoreOpener-72d273770314688e78624993de4f3398-1 {}] regionserver.HStore(327): Store=72d273770314688e78624993de4f3398/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,551 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 72d273770314688e78624993de4f3398 2024-11-13T16:08:13,552 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=28 2024-11-13T16:08:13,552 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure 4a146f98682f25e8cddcae5dbf318603, server=f8f85b0bb1d8,38187,1731514088725 in 474 msec 2024-11-13T16:08:13,552 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/72d273770314688e78624993de4f3398 2024-11-13T16:08:13,553 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/72d273770314688e78624993de4f3398 2024-11-13T16:08:13,554 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 72d273770314688e78624993de4f3398 2024-11-13T16:08:13,555 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 72d273770314688e78624993de4f3398 2024-11-13T16:08:13,555 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T16:08:13,557 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 72d273770314688e78624993de4f3398 2024-11-13T16:08:13,561 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4a146f98682f25e8cddcae5dbf318603, ASSIGN in 868 msec 2024-11-13T16:08:13,561 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731514091776.519ece296a21b5ddccc38a7c928780bb. 2024-11-13T16:08:13,561 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731514091776.519ece296a21b5ddccc38a7c928780bb. 2024-11-13T16:08:13,561 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42949 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=f8f85b0bb1d8,41333,1731514088798, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e72e57b96ff9bc0bbd862ce564d79c42. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T16:08:13,561 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42949 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=f8f85b0bb1d8,41333,1731514088798, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fe97fe0e9b37811ba99ca8100b461589. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T16:08:13,562 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731514091776.e8725a794e6d251e1bf6e3502baa7700. 2024-11-13T16:08:13,562 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => e8725a794e6d251e1bf6e3502baa7700, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731514091776.e8725a794e6d251e1bf6e3502baa7700.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-13T16:08:13,562 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true e8725a794e6d251e1bf6e3502baa7700 2024-11-13T16:08:13,562 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731514091776.e8725a794e6d251e1bf6e3502baa7700.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:13,563 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for e8725a794e6d251e1bf6e3502baa7700 2024-11-13T16:08:13,563 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for e8725a794e6d251e1bf6e3502baa7700 2024-11-13T16:08:13,565 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=519ece296a21b5ddccc38a7c928780bb, regionState=OPEN, openSeqNum=2, regionLocation=f8f85b0bb1d8,43467,1731514088560 2024-11-13T16:08:13,567 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c4a87adf770b37d40ab6691e377ad1b5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T16:08:13,568 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/72d273770314688e78624993de4f3398/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T16:08:13,568 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened c4a87adf770b37d40ab6691e377ad1b5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70974171, jitterRate=0.0575975626707077}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T16:08:13,568 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c4a87adf770b37d40ab6691e377ad1b5 2024-11-13T16:08:13,568 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for c4a87adf770b37d40ab6691e377ad1b5: Running coprocessor pre-open hook at 1731514093485Writing region info on filesystem at 1731514093485Initializing all the Stores at 1731514093495 (+10 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093495Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093503 (+8 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093503Cleaning up temporary data from old regions at 1731514093530 (+27 ms)Running coprocessor post-open hooks at 1731514093568 (+38 ms)Region opened successfully at 1731514093568 2024-11-13T16:08:13,570 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731514091776.c4a87adf770b37d40ab6691e377ad1b5., pid=49, masterSystemTime=1731514093227 2024-11-13T16:08:13,569 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 72d273770314688e78624993de4f3398; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70839017, jitterRate=0.055583611130714417}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T16:08:13,570 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 72d273770314688e78624993de4f3398 2024-11-13T16:08:13,570 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 72d273770314688e78624993de4f3398: Running coprocessor pre-open hook at 1731514093503Writing region info on filesystem at 1731514093503Initializing all the Stores at 1731514093506 (+3 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093506Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093516 (+10 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093516Cleaning up temporary data from old regions at 1731514093555 (+39 ms)Running coprocessor post-open hooks at 1731514093570 (+15 ms)Region opened successfully at 1731514093570 2024-11-13T16:08:13,572 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 519ece296a21b5ddccc38a7c928780bb, server=f8f85b0bb1d8,43467,1731514088560 because future has completed 2024-11-13T16:08:13,574 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731514091776.72d273770314688e78624993de4f3398., pid=35, masterSystemTime=1731514093089 2024-11-13T16:08:13,576 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731514091776.c4a87adf770b37d40ab6691e377ad1b5. 2024-11-13T16:08:13,576 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731514091776.c4a87adf770b37d40ab6691e377ad1b5. 2024-11-13T16:08:13,576 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731514091776.97998e7179d85b23a3f5928cd6511629. 2024-11-13T16:08:13,576 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => 97998e7179d85b23a3f5928cd6511629, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731514091776.97998e7179d85b23a3f5928cd6511629.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-13T16:08:13,577 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 97998e7179d85b23a3f5928cd6511629 2024-11-13T16:08:13,577 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731514091776.97998e7179d85b23a3f5928cd6511629.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:13,577 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for 97998e7179d85b23a3f5928cd6511629 2024-11-13T16:08:13,577 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for 97998e7179d85b23a3f5928cd6511629 2024-11-13T16:08:13,578 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731514091776.72d273770314688e78624993de4f3398. 2024-11-13T16:08:13,578 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731514091776.72d273770314688e78624993de4f3398. 2024-11-13T16:08:13,579 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731514091776.fa6636be3559bc84793534420cd4f207. 2024-11-13T16:08:13,579 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => fa6636be3559bc84793534420cd4f207, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731514091776.fa6636be3559bc84793534420cd4f207.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-13T16:08:13,579 INFO [StoreOpener-e8725a794e6d251e1bf6e3502baa7700-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region e8725a794e6d251e1bf6e3502baa7700 2024-11-13T16:08:13,579 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true fa6636be3559bc84793534420cd4f207 2024-11-13T16:08:13,579 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731514091776.fa6636be3559bc84793534420cd4f207.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:13,579 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for fa6636be3559bc84793534420cd4f207 2024-11-13T16:08:13,580 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for fa6636be3559bc84793534420cd4f207 2024-11-13T16:08:13,580 INFO [StoreOpener-97998e7179d85b23a3f5928cd6511629-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 97998e7179d85b23a3f5928cd6511629 2024-11-13T16:08:13,581 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=20 2024-11-13T16:08:13,581 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure fe97fe0e9b37811ba99ca8100b461589, server=f8f85b0bb1d8,41333,1731514088798 in 557 msec 2024-11-13T16:08:13,581 INFO [StoreOpener-e8725a794e6d251e1bf6e3502baa7700-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e8725a794e6d251e1bf6e3502baa7700 columnFamilyName colfamily11 2024-11-13T16:08:13,582 DEBUG [StoreOpener-e8725a794e6d251e1bf6e3502baa7700-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,582 INFO [StoreOpener-fa6636be3559bc84793534420cd4f207-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region fa6636be3559bc84793534420cd4f207 2024-11-13T16:08:13,583 INFO [StoreOpener-e8725a794e6d251e1bf6e3502baa7700-1 {}] regionserver.HStore(327): Store=e8725a794e6d251e1bf6e3502baa7700/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,583 INFO [StoreOpener-97998e7179d85b23a3f5928cd6511629-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97998e7179d85b23a3f5928cd6511629 columnFamilyName colfamily11 2024-11-13T16:08:13,583 INFO [StoreOpener-e8725a794e6d251e1bf6e3502baa7700-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region e8725a794e6d251e1bf6e3502baa7700 2024-11-13T16:08:13,583 DEBUG [StoreOpener-97998e7179d85b23a3f5928cd6511629-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,584 INFO [StoreOpener-97998e7179d85b23a3f5928cd6511629-1 {}] regionserver.HStore(327): Store=97998e7179d85b23a3f5928cd6511629/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,584 INFO [StoreOpener-fa6636be3559bc84793534420cd4f207-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa6636be3559bc84793534420cd4f207 columnFamilyName colfamily11 2024-11-13T16:08:13,584 DEBUG [StoreOpener-fa6636be3559bc84793534420cd4f207-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,585 INFO [StoreOpener-97998e7179d85b23a3f5928cd6511629-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 97998e7179d85b23a3f5928cd6511629 2024-11-13T16:08:13,585 INFO [StoreOpener-fa6636be3559bc84793534420cd4f207-1 {}] regionserver.HStore(327): Store=fa6636be3559bc84793534420cd4f207/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,586 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fe97fe0e9b37811ba99ca8100b461589, ASSIGN in 897 msec 2024-11-13T16:08:13,586 INFO [StoreOpener-e8725a794e6d251e1bf6e3502baa7700-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e8725a794e6d251e1bf6e3502baa7700 columnFamilyName colfamily21 2024-11-13T16:08:13,586 INFO [StoreOpener-fa6636be3559bc84793534420cd4f207-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region fa6636be3559bc84793534420cd4f207 2024-11-13T16:08:13,586 DEBUG [StoreOpener-e8725a794e6d251e1bf6e3502baa7700-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,588 INFO [StoreOpener-e8725a794e6d251e1bf6e3502baa7700-1 {}] regionserver.HStore(327): Store=e8725a794e6d251e1bf6e3502baa7700/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,588 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=72d273770314688e78624993de4f3398, regionState=OPEN, openSeqNum=2, regionLocation=f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:08:13,588 INFO [StoreOpener-e8725a794e6d251e1bf6e3502baa7700-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region e8725a794e6d251e1bf6e3502baa7700 2024-11-13T16:08:13,591 INFO [StoreOpener-fa6636be3559bc84793534420cd4f207-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa6636be3559bc84793534420cd4f207 columnFamilyName colfamily21 2024-11-13T16:08:13,591 INFO [StoreOpener-97998e7179d85b23a3f5928cd6511629-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97998e7179d85b23a3f5928cd6511629 columnFamilyName colfamily21 2024-11-13T16:08:13,591 DEBUG [StoreOpener-fa6636be3559bc84793534420cd4f207-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,593 DEBUG [StoreOpener-97998e7179d85b23a3f5928cd6511629-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,593 INFO [StoreOpener-e8725a794e6d251e1bf6e3502baa7700-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e8725a794e6d251e1bf6e3502baa7700 columnFamilyName colfamily31 2024-11-13T16:08:13,593 DEBUG [StoreOpener-e8725a794e6d251e1bf6e3502baa7700-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,594 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42949 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=f8f85b0bb1d8,43467,1731514088560, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fec5f21b48c36bf01685ec2ee58f0490. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T16:08:13,594 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42949 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=f8f85b0bb1d8,43467,1731514088560, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=519ece296a21b5ddccc38a7c928780bb. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T16:08:13,594 INFO [StoreOpener-fa6636be3559bc84793534420cd4f207-1 {}] regionserver.HStore(327): Store=fa6636be3559bc84793534420cd4f207/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,594 INFO [StoreOpener-fa6636be3559bc84793534420cd4f207-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region fa6636be3559bc84793534420cd4f207 2024-11-13T16:08:13,597 INFO [StoreOpener-fa6636be3559bc84793534420cd4f207-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa6636be3559bc84793534420cd4f207 columnFamilyName colfamily31 2024-11-13T16:08:13,597 DEBUG [StoreOpener-fa6636be3559bc84793534420cd4f207-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,598 INFO [StoreOpener-e8725a794e6d251e1bf6e3502baa7700-1 {}] regionserver.HStore(327): Store=e8725a794e6d251e1bf6e3502baa7700/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,598 INFO [StoreOpener-97998e7179d85b23a3f5928cd6511629-1 {}] regionserver.HStore(327): Store=97998e7179d85b23a3f5928cd6511629/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,598 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for e8725a794e6d251e1bf6e3502baa7700 2024-11-13T16:08:13,598 INFO [StoreOpener-97998e7179d85b23a3f5928cd6511629-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 97998e7179d85b23a3f5928cd6511629 2024-11-13T16:08:13,598 INFO [StoreOpener-fa6636be3559bc84793534420cd4f207-1 {}] regionserver.HStore(327): Store=fa6636be3559bc84793534420cd4f207/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,598 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for fa6636be3559bc84793534420cd4f207 2024-11-13T16:08:13,599 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=c4a87adf770b37d40ab6691e377ad1b5, regionState=OPEN, openSeqNum=2, regionLocation=f8f85b0bb1d8,38187,1731514088725 2024-11-13T16:08:13,599 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e8725a794e6d251e1bf6e3502baa7700 2024-11-13T16:08:13,600 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e8725a794e6d251e1bf6e3502baa7700 2024-11-13T16:08:13,600 INFO [StoreOpener-97998e7179d85b23a3f5928cd6511629-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97998e7179d85b23a3f5928cd6511629 columnFamilyName colfamily31 2024-11-13T16:08:13,601 DEBUG [StoreOpener-97998e7179d85b23a3f5928cd6511629-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,601 INFO [StoreOpener-97998e7179d85b23a3f5928cd6511629-1 {}] regionserver.HStore(327): Store=97998e7179d85b23a3f5928cd6511629/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,602 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for 97998e7179d85b23a3f5928cd6511629 2024-11-13T16:08:13,602 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for e8725a794e6d251e1bf6e3502baa7700 2024-11-13T16:08:13,602 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for e8725a794e6d251e1bf6e3502baa7700 2024-11-13T16:08:13,603 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/97998e7179d85b23a3f5928cd6511629 2024-11-13T16:08:13,603 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T16:08:13,603 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fa6636be3559bc84793534420cd4f207 2024-11-13T16:08:13,604 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/97998e7179d85b23a3f5928cd6511629 2024-11-13T16:08:13,604 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fa6636be3559bc84793534420cd4f207 2024-11-13T16:08:13,605 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for e8725a794e6d251e1bf6e3502baa7700 2024-11-13T16:08:13,605 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for 97998e7179d85b23a3f5928cd6511629 2024-11-13T16:08:13,605 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 72d273770314688e78624993de4f3398, server=f8f85b0bb1d8,41333,1731514088798 because future has completed 2024-11-13T16:08:13,605 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for 97998e7179d85b23a3f5928cd6511629 2024-11-13T16:08:13,606 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for fa6636be3559bc84793534420cd4f207 2024-11-13T16:08:13,606 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for fa6636be3559bc84793534420cd4f207 2024-11-13T16:08:13,607 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T16:08:13,608 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure c4a87adf770b37d40ab6691e377ad1b5, server=f8f85b0bb1d8,38187,1731514088725 because future has completed 2024-11-13T16:08:13,608 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T16:08:13,610 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for fa6636be3559bc84793534420cd4f207 2024-11-13T16:08:13,610 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for 97998e7179d85b23a3f5928cd6511629 2024-11-13T16:08:13,613 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e8725a794e6d251e1bf6e3502baa7700/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T16:08:13,614 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened e8725a794e6d251e1bf6e3502baa7700; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64458555, jitterRate=-0.03949268162250519}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T16:08:13,614 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e8725a794e6d251e1bf6e3502baa7700 2024-11-13T16:08:13,614 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for e8725a794e6d251e1bf6e3502baa7700: Running coprocessor pre-open hook at 1731514093563Writing region info on filesystem at 1731514093563Initializing all the Stores at 1731514093565 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093565Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093579 (+14 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093579Cleaning up temporary data from old regions at 1731514093602 (+23 ms)Running coprocessor post-open hooks at 1731514093614 (+12 ms)Region opened successfully at 1731514093614 2024-11-13T16:08:13,615 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fa6636be3559bc84793534420cd4f207/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T16:08:13,616 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731514091776.e8725a794e6d251e1bf6e3502baa7700., pid=38, masterSystemTime=1731514093033 2024-11-13T16:08:13,616 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/97998e7179d85b23a3f5928cd6511629/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T16:08:13,616 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened fa6636be3559bc84793534420cd4f207; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58970576, jitterRate=-0.12126994132995605}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T16:08:13,616 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fa6636be3559bc84793534420cd4f207 2024-11-13T16:08:13,616 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for fa6636be3559bc84793534420cd4f207: Running coprocessor pre-open hook at 1731514093580Writing region info on filesystem at 1731514093580Initializing all the Stores at 1731514093581 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093581Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093582 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093582Cleaning up temporary data from old regions at 1731514093606 (+24 ms)Running coprocessor post-open hooks at 1731514093616 (+10 ms)Region opened successfully at 1731514093616 2024-11-13T16:08:13,617 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731514091776.fa6636be3559bc84793534420cd4f207., pid=47, masterSystemTime=1731514093255 2024-11-13T16:08:13,618 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened 97998e7179d85b23a3f5928cd6511629; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60545978, jitterRate=-0.09779462218284607}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T16:08:13,618 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 97998e7179d85b23a3f5928cd6511629 2024-11-13T16:08:13,618 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for 97998e7179d85b23a3f5928cd6511629: Running coprocessor pre-open hook at 1731514093578Writing region info on filesystem at 1731514093578Initializing all the Stores at 1731514093579 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093579Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093580 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093580Cleaning up temporary data from old regions at 1731514093605 (+25 ms)Running coprocessor post-open hooks at 1731514093618 (+13 ms)Region opened successfully at 1731514093618 2024-11-13T16:08:13,619 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731514091776.97998e7179d85b23a3f5928cd6511629., pid=52, masterSystemTime=1731514093489 2024-11-13T16:08:13,620 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731514091776.e8725a794e6d251e1bf6e3502baa7700. 2024-11-13T16:08:13,620 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731514091776.e8725a794e6d251e1bf6e3502baa7700. 2024-11-13T16:08:13,621 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=25 2024-11-13T16:08:13,621 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731514091776.8ba796d80fc3b65f0948c95ccbdad39d. 2024-11-13T16:08:13,621 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure 519ece296a21b5ddccc38a7c928780bb, server=f8f85b0bb1d8,43467,1731514088560 in 610 msec 2024-11-13T16:08:13,621 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 8ba796d80fc3b65f0948c95ccbdad39d, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731514091776.8ba796d80fc3b65f0948c95ccbdad39d.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-13T16:08:13,621 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 8ba796d80fc3b65f0948c95ccbdad39d 2024-11-13T16:08:13,621 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731514091776.8ba796d80fc3b65f0948c95ccbdad39d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:13,622 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 8ba796d80fc3b65f0948c95ccbdad39d 2024-11-13T16:08:13,622 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 8ba796d80fc3b65f0948c95ccbdad39d 2024-11-13T16:08:13,624 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731514091776.97998e7179d85b23a3f5928cd6511629. 2024-11-13T16:08:13,624 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731514091776.97998e7179d85b23a3f5928cd6511629. 2024-11-13T16:08:13,626 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=519ece296a21b5ddccc38a7c928780bb, ASSIGN in 937 msec 2024-11-13T16:08:13,628 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=97998e7179d85b23a3f5928cd6511629, regionState=OPEN, openSeqNum=2, regionLocation=f8f85b0bb1d8,38187,1731514088725 2024-11-13T16:08:13,628 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731514091776.fa6636be3559bc84793534420cd4f207. 2024-11-13T16:08:13,628 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731514091776.fa6636be3559bc84793534420cd4f207. 2024-11-13T16:08:13,628 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731514091776.05d65f4a6bfecb3d530d15832c218252. 2024-11-13T16:08:13,628 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => 05d65f4a6bfecb3d530d15832c218252, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731514091776.05d65f4a6bfecb3d530d15832c218252.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-13T16:08:13,629 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 05d65f4a6bfecb3d530d15832c218252 2024-11-13T16:08:13,629 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731514091776.05d65f4a6bfecb3d530d15832c218252.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:13,629 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for 05d65f4a6bfecb3d530d15832c218252 2024-11-13T16:08:13,629 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for 05d65f4a6bfecb3d530d15832c218252 2024-11-13T16:08:13,632 INFO [StoreOpener-05d65f4a6bfecb3d530d15832c218252-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 05d65f4a6bfecb3d530d15832c218252 2024-11-13T16:08:13,632 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 97998e7179d85b23a3f5928cd6511629, server=f8f85b0bb1d8,38187,1731514088725 because future has completed 2024-11-13T16:08:13,633 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=fa6636be3559bc84793534420cd4f207, regionState=OPEN, openSeqNum=2, regionLocation=f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:08:13,634 INFO [StoreOpener-05d65f4a6bfecb3d530d15832c218252-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 05d65f4a6bfecb3d530d15832c218252 columnFamilyName colfamily11 2024-11-13T16:08:13,634 DEBUG [StoreOpener-05d65f4a6bfecb3d530d15832c218252-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,635 INFO [StoreOpener-05d65f4a6bfecb3d530d15832c218252-1 {}] regionserver.HStore(327): Store=05d65f4a6bfecb3d530d15832c218252/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,635 INFO [StoreOpener-05d65f4a6bfecb3d530d15832c218252-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 05d65f4a6bfecb3d530d15832c218252 2024-11-13T16:08:13,631 INFO [StoreOpener-8ba796d80fc3b65f0948c95ccbdad39d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 8ba796d80fc3b65f0948c95ccbdad39d 2024-11-13T16:08:13,638 INFO [StoreOpener-05d65f4a6bfecb3d530d15832c218252-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 05d65f4a6bfecb3d530d15832c218252 columnFamilyName colfamily21 2024-11-13T16:08:13,639 DEBUG [StoreOpener-05d65f4a6bfecb3d530d15832c218252-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,639 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa6636be3559bc84793534420cd4f207, server=f8f85b0bb1d8,41333,1731514088798 because future has completed 2024-11-13T16:08:13,640 INFO [StoreOpener-05d65f4a6bfecb3d530d15832c218252-1 {}] regionserver.HStore(327): Store=05d65f4a6bfecb3d530d15832c218252/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,640 INFO [StoreOpener-8ba796d80fc3b65f0948c95ccbdad39d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8ba796d80fc3b65f0948c95ccbdad39d columnFamilyName colfamily11 2024-11-13T16:08:13,640 DEBUG [StoreOpener-8ba796d80fc3b65f0948c95ccbdad39d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,640 INFO [StoreOpener-05d65f4a6bfecb3d530d15832c218252-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 05d65f4a6bfecb3d530d15832c218252 2024-11-13T16:08:13,642 INFO [StoreOpener-8ba796d80fc3b65f0948c95ccbdad39d-1 {}] regionserver.HStore(327): Store=8ba796d80fc3b65f0948c95ccbdad39d/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,642 INFO [StoreOpener-8ba796d80fc3b65f0948c95ccbdad39d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 8ba796d80fc3b65f0948c95ccbdad39d 2024-11-13T16:08:13,644 INFO [StoreOpener-8ba796d80fc3b65f0948c95ccbdad39d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8ba796d80fc3b65f0948c95ccbdad39d columnFamilyName colfamily21 2024-11-13T16:08:13,644 INFO [StoreOpener-05d65f4a6bfecb3d530d15832c218252-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 05d65f4a6bfecb3d530d15832c218252 columnFamilyName colfamily31 2024-11-13T16:08:13,644 DEBUG [StoreOpener-8ba796d80fc3b65f0948c95ccbdad39d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,644 DEBUG [StoreOpener-05d65f4a6bfecb3d530d15832c218252-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,645 INFO [StoreOpener-8ba796d80fc3b65f0948c95ccbdad39d-1 {}] regionserver.HStore(327): Store=8ba796d80fc3b65f0948c95ccbdad39d/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,645 INFO [StoreOpener-05d65f4a6bfecb3d530d15832c218252-1 {}] regionserver.HStore(327): Store=05d65f4a6bfecb3d530d15832c218252/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,645 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for 05d65f4a6bfecb3d530d15832c218252 2024-11-13T16:08:13,645 INFO [StoreOpener-8ba796d80fc3b65f0948c95ccbdad39d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 8ba796d80fc3b65f0948c95ccbdad39d 2024-11-13T16:08:13,646 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/05d65f4a6bfecb3d530d15832c218252 2024-11-13T16:08:13,647 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/05d65f4a6bfecb3d530d15832c218252 2024-11-13T16:08:13,648 INFO [StoreOpener-8ba796d80fc3b65f0948c95ccbdad39d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8ba796d80fc3b65f0948c95ccbdad39d columnFamilyName colfamily31 2024-11-13T16:08:13,648 DEBUG [StoreOpener-8ba796d80fc3b65f0948c95ccbdad39d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,648 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for 05d65f4a6bfecb3d530d15832c218252 2024-11-13T16:08:13,648 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for 05d65f4a6bfecb3d530d15832c218252 2024-11-13T16:08:13,649 INFO [StoreOpener-8ba796d80fc3b65f0948c95ccbdad39d-1 {}] regionserver.HStore(327): Store=8ba796d80fc3b65f0948c95ccbdad39d/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,649 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 8ba796d80fc3b65f0948c95ccbdad39d 2024-11-13T16:08:13,649 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T16:08:13,650 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8ba796d80fc3b65f0948c95ccbdad39d 2024-11-13T16:08:13,650 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=24 2024-11-13T16:08:13,650 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure 97998e7179d85b23a3f5928cd6511629, server=f8f85b0bb1d8,38187,1731514088725 in 329 msec 2024-11-13T16:08:13,651 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8ba796d80fc3b65f0948c95ccbdad39d 2024-11-13T16:08:13,651 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for 05d65f4a6bfecb3d530d15832c218252 2024-11-13T16:08:13,652 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 8ba796d80fc3b65f0948c95ccbdad39d 2024-11-13T16:08:13,652 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 8ba796d80fc3b65f0948c95ccbdad39d 2024-11-13T16:08:13,653 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T16:08:13,655 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/05d65f4a6bfecb3d530d15832c218252/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T16:08:13,655 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 8ba796d80fc3b65f0948c95ccbdad39d 2024-11-13T16:08:13,655 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=97998e7179d85b23a3f5928cd6511629, ASSIGN in 967 msec 2024-11-13T16:08:13,655 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened 05d65f4a6bfecb3d530d15832c218252; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69276490, jitterRate=0.03230014443397522}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T16:08:13,655 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 05d65f4a6bfecb3d530d15832c218252 2024-11-13T16:08:13,656 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for 05d65f4a6bfecb3d530d15832c218252: Running coprocessor pre-open hook at 1731514093629Writing region info on filesystem at 1731514093629Initializing all the Stores at 1731514093631 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093631Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093631Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093631Cleaning up temporary data from old regions at 1731514093648 (+17 ms)Running coprocessor post-open hooks at 1731514093655 (+7 ms)Region opened successfully at 1731514093655 2024-11-13T16:08:13,657 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731514091776.05d65f4a6bfecb3d530d15832c218252., pid=48, masterSystemTime=1731514093255 2024-11-13T16:08:13,660 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8ba796d80fc3b65f0948c95ccbdad39d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T16:08:13,661 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 8ba796d80fc3b65f0948c95ccbdad39d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65091636, jitterRate=-0.03005903959274292}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T16:08:13,662 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8ba796d80fc3b65f0948c95ccbdad39d 2024-11-13T16:08:13,662 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 8ba796d80fc3b65f0948c95ccbdad39d: Running coprocessor pre-open hook at 1731514093622Writing region info on filesystem at 1731514093622Initializing all the Stores at 1731514093624 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093624Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093631 (+7 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093631Cleaning up temporary data from old regions at 1731514093652 (+21 ms)Running coprocessor post-open hooks at 1731514093662 (+10 ms)Region opened successfully at 1731514093662 2024-11-13T16:08:13,666 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731514091776.8ba796d80fc3b65f0948c95ccbdad39d., pid=36, masterSystemTime=1731514093033 2024-11-13T16:08:13,668 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42949 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=f8f85b0bb1d8,38187,1731514088725, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c4a87adf770b37d40ab6691e377ad1b5. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T16:08:13,669 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731514091776.05d65f4a6bfecb3d530d15832c218252. 2024-11-13T16:08:13,669 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731514091776.05d65f4a6bfecb3d530d15832c218252. 2024-11-13T16:08:13,669 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731514091776.1fcf78578ccab75f1705f3c2d7bcdb14. 2024-11-13T16:08:13,669 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => 1fcf78578ccab75f1705f3c2d7bcdb14, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731514091776.1fcf78578ccab75f1705f3c2d7bcdb14.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-13T16:08:13,670 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 1fcf78578ccab75f1705f3c2d7bcdb14 2024-11-13T16:08:13,670 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42949 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=f8f85b0bb1d8,41333,1731514088798, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e72e57b96ff9bc0bbd862ce564d79c42. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T16:08:13,670 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731514091776.1fcf78578ccab75f1705f3c2d7bcdb14.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:13,670 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42949 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=f8f85b0bb1d8,41333,1731514088798, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=72d273770314688e78624993de4f3398. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T16:08:13,670 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for 1fcf78578ccab75f1705f3c2d7bcdb14 2024-11-13T16:08:13,670 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for 1fcf78578ccab75f1705f3c2d7bcdb14 2024-11-13T16:08:13,674 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731514091776.8ba796d80fc3b65f0948c95ccbdad39d. 2024-11-13T16:08:13,674 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731514091776.8ba796d80fc3b65f0948c95ccbdad39d. 2024-11-13T16:08:13,675 INFO [StoreOpener-1fcf78578ccab75f1705f3c2d7bcdb14-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 1fcf78578ccab75f1705f3c2d7bcdb14 2024-11-13T16:08:13,676 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=14 2024-11-13T16:08:13,676 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure fa6636be3559bc84793534420cd4f207, server=f8f85b0bb1d8,41333,1731514088798 in 552 msec 2024-11-13T16:08:13,681 INFO [StoreOpener-1fcf78578ccab75f1705f3c2d7bcdb14-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1fcf78578ccab75f1705f3c2d7bcdb14 columnFamilyName colfamily11 2024-11-13T16:08:13,681 DEBUG [StoreOpener-1fcf78578ccab75f1705f3c2d7bcdb14-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,681 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fa6636be3559bc84793534420cd4f207, ASSIGN in 993 msec 2024-11-13T16:08:13,682 INFO [StoreOpener-1fcf78578ccab75f1705f3c2d7bcdb14-1 {}] regionserver.HStore(327): Store=1fcf78578ccab75f1705f3c2d7bcdb14/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,683 INFO [StoreOpener-1fcf78578ccab75f1705f3c2d7bcdb14-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 1fcf78578ccab75f1705f3c2d7bcdb14 2024-11-13T16:08:13,683 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=8ba796d80fc3b65f0948c95ccbdad39d, regionState=OPEN, openSeqNum=2, regionLocation=f8f85b0bb1d8,43467,1731514088560 2024-11-13T16:08:13,687 INFO [StoreOpener-1fcf78578ccab75f1705f3c2d7bcdb14-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1fcf78578ccab75f1705f3c2d7bcdb14 columnFamilyName colfamily21 2024-11-13T16:08:13,687 DEBUG [StoreOpener-1fcf78578ccab75f1705f3c2d7bcdb14-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,688 INFO [StoreOpener-1fcf78578ccab75f1705f3c2d7bcdb14-1 {}] regionserver.HStore(327): Store=1fcf78578ccab75f1705f3c2d7bcdb14/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,688 INFO [StoreOpener-1fcf78578ccab75f1705f3c2d7bcdb14-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 1fcf78578ccab75f1705f3c2d7bcdb14 2024-11-13T16:08:13,688 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=05d65f4a6bfecb3d530d15832c218252, regionState=OPEN, openSeqNum=2, regionLocation=f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:08:13,691 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ba796d80fc3b65f0948c95ccbdad39d, server=f8f85b0bb1d8,43467,1731514088560 because future has completed 2024-11-13T16:08:13,693 INFO [StoreOpener-1fcf78578ccab75f1705f3c2d7bcdb14-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1fcf78578ccab75f1705f3c2d7bcdb14 columnFamilyName colfamily31 2024-11-13T16:08:13,693 DEBUG [StoreOpener-1fcf78578ccab75f1705f3c2d7bcdb14-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:13,693 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 05d65f4a6bfecb3d530d15832c218252, server=f8f85b0bb1d8,41333,1731514088798 because future has completed 2024-11-13T16:08:13,694 INFO [StoreOpener-1fcf78578ccab75f1705f3c2d7bcdb14-1 {}] regionserver.HStore(327): Store=1fcf78578ccab75f1705f3c2d7bcdb14/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:13,695 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for 1fcf78578ccab75f1705f3c2d7bcdb14 2024-11-13T16:08:13,696 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1fcf78578ccab75f1705f3c2d7bcdb14 2024-11-13T16:08:13,697 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1fcf78578ccab75f1705f3c2d7bcdb14 2024-11-13T16:08:13,700 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42949 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=f8f85b0bb1d8,43467,1731514088560, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fec5f21b48c36bf01685ec2ee58f0490. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T16:08:13,700 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for 1fcf78578ccab75f1705f3c2d7bcdb14 2024-11-13T16:08:13,700 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for 1fcf78578ccab75f1705f3c2d7bcdb14 2024-11-13T16:08:13,702 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T16:08:13,705 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for 1fcf78578ccab75f1705f3c2d7bcdb14 2024-11-13T16:08:13,710 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=6 2024-11-13T16:08:13,710 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure 8ba796d80fc3b65f0948c95ccbdad39d, server=f8f85b0bb1d8,43467,1731514088560 in 744 msec 2024-11-13T16:08:13,722 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1fcf78578ccab75f1705f3c2d7bcdb14/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T16:08:13,723 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened 1fcf78578ccab75f1705f3c2d7bcdb14; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61713207, jitterRate=-0.08040155470371246}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T16:08:13,724 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1fcf78578ccab75f1705f3c2d7bcdb14 2024-11-13T16:08:13,724 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8ba796d80fc3b65f0948c95ccbdad39d, ASSIGN in 1.0260 sec 2024-11-13T16:08:13,724 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for 1fcf78578ccab75f1705f3c2d7bcdb14: Running coprocessor pre-open hook at 1731514093671Writing region info on filesystem at 1731514093671Initializing all the Stores at 1731514093673 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093673Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093673Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514093673Cleaning up temporary data from old regions at 1731514093701 (+28 ms)Running coprocessor post-open hooks at 1731514093724 (+23 ms)Region opened successfully at 1731514093724 2024-11-13T16:08:13,726 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731514091776.1fcf78578ccab75f1705f3c2d7bcdb14., pid=46, masterSystemTime=1731514093255 2024-11-13T16:08:13,732 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731514091776.1fcf78578ccab75f1705f3c2d7bcdb14. 2024-11-13T16:08:13,733 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731514091776.1fcf78578ccab75f1705f3c2d7bcdb14. 2024-11-13T16:08:13,743 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=15 2024-11-13T16:08:13,743 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure 05d65f4a6bfecb3d530d15832c218252, server=f8f85b0bb1d8,41333,1731514088798 in 608 msec 2024-11-13T16:08:13,755 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=05d65f4a6bfecb3d530d15832c218252, ASSIGN in 1.0590 sec 2024-11-13T16:08:13,757 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=1fcf78578ccab75f1705f3c2d7bcdb14, regionState=OPEN, openSeqNum=2, regionLocation=f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:08:13,769 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=e8725a794e6d251e1bf6e3502baa7700, regionState=OPEN, openSeqNum=2, regionLocation=f8f85b0bb1d8,43467,1731514088560 2024-11-13T16:08:13,773 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1fcf78578ccab75f1705f3c2d7bcdb14, server=f8f85b0bb1d8,41333,1731514088798 because future has completed 2024-11-13T16:08:13,774 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42949 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=f8f85b0bb1d8,38187,1731514088725, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c4a87adf770b37d40ab6691e377ad1b5. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T16:08:13,779 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure e8725a794e6d251e1bf6e3502baa7700, server=f8f85b0bb1d8,43467,1731514088560 because future has completed 2024-11-13T16:08:13,779 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42949 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=f8f85b0bb1d8,41333,1731514088798, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e72e57b96ff9bc0bbd862ce564d79c42. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T16:08:13,779 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42949 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=f8f85b0bb1d8,41333,1731514088798, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=72d273770314688e78624993de4f3398. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T16:08:13,792 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=11 2024-11-13T16:08:13,793 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 1fcf78578ccab75f1705f3c2d7bcdb14, server=f8f85b0bb1d8,41333,1731514088798 in 685 msec 2024-11-13T16:08:13,796 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1fcf78578ccab75f1705f3c2d7bcdb14, ASSIGN in 1.1090 sec 2024-11-13T16:08:13,803 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=13 2024-11-13T16:08:13,803 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure e8725a794e6d251e1bf6e3502baa7700, server=f8f85b0bb1d8,43467,1731514088560 in 829 msec 2024-11-13T16:08:13,806 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42949 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=f8f85b0bb1d8,43467,1731514088560, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fec5f21b48c36bf01685ec2ee58f0490. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T16:08:13,811 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e8725a794e6d251e1bf6e3502baa7700, ASSIGN in 1.1190 sec 2024-11-13T16:08:13,826 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=26 2024-11-13T16:08:13,826 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure c4a87adf770b37d40ab6691e377ad1b5, server=f8f85b0bb1d8,38187,1731514088725 in 676 msec 2024-11-13T16:08:13,835 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c4a87adf770b37d40ab6691e377ad1b5, ASSIGN in 1.1430 sec 2024-11-13T16:08:13,841 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=30 2024-11-13T16:08:13,842 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure 72d273770314688e78624993de4f3398, server=f8f85b0bb1d8,41333,1731514088798 in 895 msec 2024-11-13T16:08:13,849 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=72d273770314688e78624993de4f3398, ASSIGN in 1.1580 sec 2024-11-13T16:08:13,862 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=23 2024-11-13T16:08:13,862 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure fec5f21b48c36bf01685ec2ee58f0490, server=f8f85b0bb1d8,43467,1731514088560 in 860 msec 2024-11-13T16:08:13,867 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fec5f21b48c36bf01685ec2ee58f0490, ASSIGN in 1.1790 sec 2024-11-13T16:08:13,869 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=aa124b802c708ca1f715cb1f8c9c4eaa, regionState=OPEN, openSeqNum=2, regionLocation=f8f85b0bb1d8,43467,1731514088560 2024-11-13T16:08:13,873 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure aa124b802c708ca1f715cb1f8c9c4eaa, server=f8f85b0bb1d8,43467,1731514088560 because future has completed 2024-11-13T16:08:13,881 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=8 2024-11-13T16:08:13,881 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure e72e57b96ff9bc0bbd862ce564d79c42, server=f8f85b0bb1d8,41333,1731514088798 in 843 msec 2024-11-13T16:08:13,893 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e72e57b96ff9bc0bbd862ce564d79c42, ASSIGN in 1.1970 sec 2024-11-13T16:08:13,912 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=17 2024-11-13T16:08:13,912 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure aa124b802c708ca1f715cb1f8c9c4eaa, server=f8f85b0bb1d8,43467,1731514088560 in 1.0010 sec 2024-11-13T16:08:13,921 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=aa124b802c708ca1f715cb1f8c9c4eaa, ASSIGN in 1.2280 sec 2024-11-13T16:08:13,921 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=78285ca993cc7fbfd78041c30558f756, regionState=OPENING, regionLocation=f8f85b0bb1d8,38187,1731514088725 2024-11-13T16:08:13,922 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=1a610c2403a0b491d368473a6e1fb049, regionState=OPENING, regionLocation=f8f85b0bb1d8,38187,1731514088725 2024-11-13T16:08:13,923 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=34f3d3b6a6dae1f4077567bed0bd8e75, regionState=OPENING, regionLocation=f8f85b0bb1d8,38187,1731514088725 2024-11-13T16:08:13,924 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=93f8ed38b68bd013eb0cf0e7615013ae, regionState=OPENING, regionLocation=f8f85b0bb1d8,38187,1731514088725 2024-11-13T16:08:13,926 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=78285ca993cc7fbfd78041c30558f756, ASSIGN because future has completed 2024-11-13T16:08:13,926 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 78285ca993cc7fbfd78041c30558f756, server=f8f85b0bb1d8,38187,1731514088725}] 2024-11-13T16:08:13,928 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1a610c2403a0b491d368473a6e1fb049, ASSIGN because future has completed 2024-11-13T16:08:13,929 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=34f3d3b6a6dae1f4077567bed0bd8e75, ASSIGN because future has completed 2024-11-13T16:08:13,931 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=93f8ed38b68bd013eb0cf0e7615013ae, ASSIGN because future has completed 2024-11-13T16:08:13,933 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 93f8ed38b68bd013eb0cf0e7615013ae, server=f8f85b0bb1d8,38187,1731514088725}] 2024-11-13T16:08:13,938 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 34f3d3b6a6dae1f4077567bed0bd8e75, server=f8f85b0bb1d8,38187,1731514088725}] 2024-11-13T16:08:13,955 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a610c2403a0b491d368473a6e1fb049, server=f8f85b0bb1d8,38187,1731514088725}] 2024-11-13T16:08:13,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T16:08:14,089 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731514091776.34f3d3b6a6dae1f4077567bed0bd8e75. 2024-11-13T16:08:14,089 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => 34f3d3b6a6dae1f4077567bed0bd8e75, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731514091776.34f3d3b6a6dae1f4077567bed0bd8e75.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-13T16:08:14,090 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 34f3d3b6a6dae1f4077567bed0bd8e75 2024-11-13T16:08:14,090 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731514091776.34f3d3b6a6dae1f4077567bed0bd8e75.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:14,090 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for 34f3d3b6a6dae1f4077567bed0bd8e75 2024-11-13T16:08:14,090 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for 34f3d3b6a6dae1f4077567bed0bd8e75 2024-11-13T16:08:14,092 INFO [StoreOpener-34f3d3b6a6dae1f4077567bed0bd8e75-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 34f3d3b6a6dae1f4077567bed0bd8e75 2024-11-13T16:08:14,095 INFO [StoreOpener-34f3d3b6a6dae1f4077567bed0bd8e75-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 34f3d3b6a6dae1f4077567bed0bd8e75 columnFamilyName colfamily11 2024-11-13T16:08:14,095 DEBUG [StoreOpener-34f3d3b6a6dae1f4077567bed0bd8e75-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:14,095 INFO [StoreOpener-34f3d3b6a6dae1f4077567bed0bd8e75-1 {}] regionserver.HStore(327): Store=34f3d3b6a6dae1f4077567bed0bd8e75/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:14,096 INFO [StoreOpener-34f3d3b6a6dae1f4077567bed0bd8e75-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 34f3d3b6a6dae1f4077567bed0bd8e75 2024-11-13T16:08:14,097 INFO [StoreOpener-34f3d3b6a6dae1f4077567bed0bd8e75-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 34f3d3b6a6dae1f4077567bed0bd8e75 columnFamilyName colfamily21 2024-11-13T16:08:14,097 DEBUG [StoreOpener-34f3d3b6a6dae1f4077567bed0bd8e75-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:14,098 INFO [StoreOpener-34f3d3b6a6dae1f4077567bed0bd8e75-1 {}] regionserver.HStore(327): Store=34f3d3b6a6dae1f4077567bed0bd8e75/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:14,098 INFO [StoreOpener-34f3d3b6a6dae1f4077567bed0bd8e75-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 34f3d3b6a6dae1f4077567bed0bd8e75 2024-11-13T16:08:14,100 INFO [StoreOpener-34f3d3b6a6dae1f4077567bed0bd8e75-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 34f3d3b6a6dae1f4077567bed0bd8e75 columnFamilyName colfamily31 2024-11-13T16:08:14,100 DEBUG [StoreOpener-34f3d3b6a6dae1f4077567bed0bd8e75-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:14,101 INFO [StoreOpener-34f3d3b6a6dae1f4077567bed0bd8e75-1 {}] regionserver.HStore(327): Store=34f3d3b6a6dae1f4077567bed0bd8e75/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:14,101 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for 34f3d3b6a6dae1f4077567bed0bd8e75 2024-11-13T16:08:14,102 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/34f3d3b6a6dae1f4077567bed0bd8e75 2024-11-13T16:08:14,103 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/34f3d3b6a6dae1f4077567bed0bd8e75 2024-11-13T16:08:14,104 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for 34f3d3b6a6dae1f4077567bed0bd8e75 2024-11-13T16:08:14,105 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for 34f3d3b6a6dae1f4077567bed0bd8e75 2024-11-13T16:08:14,105 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T16:08:14,107 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for 34f3d3b6a6dae1f4077567bed0bd8e75 2024-11-13T16:08:14,110 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/34f3d3b6a6dae1f4077567bed0bd8e75/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T16:08:14,111 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened 34f3d3b6a6dae1f4077567bed0bd8e75; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60621000, jitterRate=-0.09667670726776123}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T16:08:14,111 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 34f3d3b6a6dae1f4077567bed0bd8e75 2024-11-13T16:08:14,111 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for 34f3d3b6a6dae1f4077567bed0bd8e75: Running coprocessor pre-open hook at 1731514094090Writing region info on filesystem at 1731514094090Initializing all the Stores at 1731514094092 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514094092Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514094092Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514094093 (+1 ms)Cleaning up temporary data from old regions at 1731514094105 (+12 ms)Running coprocessor post-open hooks at 1731514094111 (+6 ms)Region opened successfully at 1731514094111 2024-11-13T16:08:14,113 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731514091776.34f3d3b6a6dae1f4077567bed0bd8e75., pid=55, masterSystemTime=1731514094081 2024-11-13T16:08:14,121 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731514091776.34f3d3b6a6dae1f4077567bed0bd8e75. 2024-11-13T16:08:14,121 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731514091776.34f3d3b6a6dae1f4077567bed0bd8e75. 2024-11-13T16:08:14,122 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731514091776.93f8ed38b68bd013eb0cf0e7615013ae. 2024-11-13T16:08:14,122 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => 93f8ed38b68bd013eb0cf0e7615013ae, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731514091776.93f8ed38b68bd013eb0cf0e7615013ae.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-13T16:08:14,123 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 93f8ed38b68bd013eb0cf0e7615013ae 2024-11-13T16:08:14,123 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731514091776.93f8ed38b68bd013eb0cf0e7615013ae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:14,123 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for 93f8ed38b68bd013eb0cf0e7615013ae 2024-11-13T16:08:14,123 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for 93f8ed38b68bd013eb0cf0e7615013ae 2024-11-13T16:08:14,128 INFO [StoreOpener-93f8ed38b68bd013eb0cf0e7615013ae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 93f8ed38b68bd013eb0cf0e7615013ae 2024-11-13T16:08:14,130 INFO [StoreOpener-93f8ed38b68bd013eb0cf0e7615013ae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 93f8ed38b68bd013eb0cf0e7615013ae columnFamilyName colfamily11 2024-11-13T16:08:14,130 DEBUG [StoreOpener-93f8ed38b68bd013eb0cf0e7615013ae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:14,132 INFO [StoreOpener-93f8ed38b68bd013eb0cf0e7615013ae-1 {}] regionserver.HStore(327): Store=93f8ed38b68bd013eb0cf0e7615013ae/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:14,132 INFO [StoreOpener-93f8ed38b68bd013eb0cf0e7615013ae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 93f8ed38b68bd013eb0cf0e7615013ae 2024-11-13T16:08:14,136 INFO [StoreOpener-93f8ed38b68bd013eb0cf0e7615013ae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 93f8ed38b68bd013eb0cf0e7615013ae columnFamilyName colfamily21 2024-11-13T16:08:14,137 DEBUG [StoreOpener-93f8ed38b68bd013eb0cf0e7615013ae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:14,139 INFO [StoreOpener-93f8ed38b68bd013eb0cf0e7615013ae-1 {}] regionserver.HStore(327): Store=93f8ed38b68bd013eb0cf0e7615013ae/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:14,139 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=34f3d3b6a6dae1f4077567bed0bd8e75, regionState=OPEN, openSeqNum=2, regionLocation=f8f85b0bb1d8,38187,1731514088725 2024-11-13T16:08:14,141 INFO [StoreOpener-93f8ed38b68bd013eb0cf0e7615013ae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 93f8ed38b68bd013eb0cf0e7615013ae 2024-11-13T16:08:14,145 INFO [StoreOpener-93f8ed38b68bd013eb0cf0e7615013ae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 93f8ed38b68bd013eb0cf0e7615013ae columnFamilyName colfamily31 2024-11-13T16:08:14,145 DEBUG [StoreOpener-93f8ed38b68bd013eb0cf0e7615013ae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:14,146 INFO [StoreOpener-93f8ed38b68bd013eb0cf0e7615013ae-1 {}] regionserver.HStore(327): Store=93f8ed38b68bd013eb0cf0e7615013ae/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:14,146 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for 93f8ed38b68bd013eb0cf0e7615013ae 2024-11-13T16:08:14,147 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/93f8ed38b68bd013eb0cf0e7615013ae 2024-11-13T16:08:14,148 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/93f8ed38b68bd013eb0cf0e7615013ae 2024-11-13T16:08:14,151 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for 93f8ed38b68bd013eb0cf0e7615013ae 2024-11-13T16:08:14,151 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for 93f8ed38b68bd013eb0cf0e7615013ae 2024-11-13T16:08:14,152 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T16:08:14,154 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for 93f8ed38b68bd013eb0cf0e7615013ae 2024-11-13T16:08:14,155 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 34f3d3b6a6dae1f4077567bed0bd8e75, server=f8f85b0bb1d8,38187,1731514088725 because future has completed 2024-11-13T16:08:14,165 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/93f8ed38b68bd013eb0cf0e7615013ae/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T16:08:14,169 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened 93f8ed38b68bd013eb0cf0e7615013ae; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72146673, jitterRate=0.07506920397281647}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T16:08:14,170 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 93f8ed38b68bd013eb0cf0e7615013ae 2024-11-13T16:08:14,170 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for 93f8ed38b68bd013eb0cf0e7615013ae: Running coprocessor pre-open hook at 1731514094124Writing region info on filesystem at 1731514094124Initializing all the Stores at 1731514094127 (+3 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514094127Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514094128 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514094128Cleaning up temporary data from old regions at 1731514094151 (+23 ms)Running coprocessor post-open hooks at 1731514094170 (+19 ms)Region opened successfully at 1731514094170 2024-11-13T16:08:14,171 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731514091776.93f8ed38b68bd013eb0cf0e7615013ae., pid=54, masterSystemTime=1731514094081 2024-11-13T16:08:14,181 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731514091776.93f8ed38b68bd013eb0cf0e7615013ae. 2024-11-13T16:08:14,181 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=9 2024-11-13T16:08:14,181 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731514091776.93f8ed38b68bd013eb0cf0e7615013ae. 2024-11-13T16:08:14,181 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure 34f3d3b6a6dae1f4077567bed0bd8e75, server=f8f85b0bb1d8,38187,1731514088725 in 225 msec 2024-11-13T16:08:14,181 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731514091776.1a610c2403a0b491d368473a6e1fb049. 2024-11-13T16:08:14,182 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => 1a610c2403a0b491d368473a6e1fb049, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731514091776.1a610c2403a0b491d368473a6e1fb049.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-13T16:08:14,182 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 1a610c2403a0b491d368473a6e1fb049 2024-11-13T16:08:14,182 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731514091776.1a610c2403a0b491d368473a6e1fb049.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:14,183 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for 1a610c2403a0b491d368473a6e1fb049 2024-11-13T16:08:14,183 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for 1a610c2403a0b491d368473a6e1fb049 2024-11-13T16:08:14,192 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=34f3d3b6a6dae1f4077567bed0bd8e75, ASSIGN in 1.4970 sec 2024-11-13T16:08:14,195 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42949 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=f8f85b0bb1d8,38187,1731514088725, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=93f8ed38b68bd013eb0cf0e7615013ae. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T16:08:14,196 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=93f8ed38b68bd013eb0cf0e7615013ae, regionState=OPEN, openSeqNum=2, regionLocation=f8f85b0bb1d8,38187,1731514088725 2024-11-13T16:08:14,199 INFO [StoreOpener-1a610c2403a0b491d368473a6e1fb049-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 1a610c2403a0b491d368473a6e1fb049 2024-11-13T16:08:14,201 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 93f8ed38b68bd013eb0cf0e7615013ae, server=f8f85b0bb1d8,38187,1731514088725 because future has completed 2024-11-13T16:08:14,201 INFO [StoreOpener-1a610c2403a0b491d368473a6e1fb049-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a610c2403a0b491d368473a6e1fb049 columnFamilyName colfamily11 2024-11-13T16:08:14,202 DEBUG [StoreOpener-1a610c2403a0b491d368473a6e1fb049-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:14,203 INFO [StoreOpener-1a610c2403a0b491d368473a6e1fb049-1 {}] regionserver.HStore(327): Store=1a610c2403a0b491d368473a6e1fb049/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:14,203 INFO [StoreOpener-1a610c2403a0b491d368473a6e1fb049-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 1a610c2403a0b491d368473a6e1fb049 2024-11-13T16:08:14,205 INFO [StoreOpener-1a610c2403a0b491d368473a6e1fb049-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a610c2403a0b491d368473a6e1fb049 columnFamilyName colfamily21 2024-11-13T16:08:14,205 DEBUG [StoreOpener-1a610c2403a0b491d368473a6e1fb049-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:14,206 INFO [StoreOpener-1a610c2403a0b491d368473a6e1fb049-1 {}] regionserver.HStore(327): Store=1a610c2403a0b491d368473a6e1fb049/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:14,206 INFO [StoreOpener-1a610c2403a0b491d368473a6e1fb049-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 1a610c2403a0b491d368473a6e1fb049 2024-11-13T16:08:14,208 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=7 2024-11-13T16:08:14,208 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 93f8ed38b68bd013eb0cf0e7615013ae, server=f8f85b0bb1d8,38187,1731514088725 in 270 msec 2024-11-13T16:08:14,214 INFO [StoreOpener-1a610c2403a0b491d368473a6e1fb049-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a610c2403a0b491d368473a6e1fb049 columnFamilyName colfamily31 2024-11-13T16:08:14,214 DEBUG [StoreOpener-1a610c2403a0b491d368473a6e1fb049-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:14,214 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=93f8ed38b68bd013eb0cf0e7615013ae, ASSIGN in 1.5240 sec 2024-11-13T16:08:14,215 INFO [StoreOpener-1a610c2403a0b491d368473a6e1fb049-1 {}] regionserver.HStore(327): Store=1a610c2403a0b491d368473a6e1fb049/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:14,215 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for 1a610c2403a0b491d368473a6e1fb049 2024-11-13T16:08:14,217 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1a610c2403a0b491d368473a6e1fb049 2024-11-13T16:08:14,218 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1a610c2403a0b491d368473a6e1fb049 2024-11-13T16:08:14,221 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for 1a610c2403a0b491d368473a6e1fb049 2024-11-13T16:08:14,221 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for 1a610c2403a0b491d368473a6e1fb049 2024-11-13T16:08:14,222 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T16:08:14,228 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for 1a610c2403a0b491d368473a6e1fb049 2024-11-13T16:08:14,233 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1a610c2403a0b491d368473a6e1fb049/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T16:08:14,239 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened 1a610c2403a0b491d368473a6e1fb049; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65378823, jitterRate=-0.025779619812965393}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T16:08:14,239 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1a610c2403a0b491d368473a6e1fb049 2024-11-13T16:08:14,239 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for 1a610c2403a0b491d368473a6e1fb049: Running coprocessor pre-open hook at 1731514094183Writing region info on filesystem at 1731514094183Initializing all the Stores at 1731514094185 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514094185Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514094198 (+13 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514094199 (+1 ms)Cleaning up temporary data from old regions at 1731514094221 (+22 ms)Running coprocessor post-open hooks at 1731514094239 (+18 ms)Region opened successfully at 1731514094239 2024-11-13T16:08:14,241 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731514091776.1a610c2403a0b491d368473a6e1fb049., pid=56, masterSystemTime=1731514094081 2024-11-13T16:08:14,244 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731514091776.1a610c2403a0b491d368473a6e1fb049. 2024-11-13T16:08:14,244 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731514091776.1a610c2403a0b491d368473a6e1fb049. 2024-11-13T16:08:14,244 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731514091776.78285ca993cc7fbfd78041c30558f756. 2024-11-13T16:08:14,245 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => 78285ca993cc7fbfd78041c30558f756, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731514091776.78285ca993cc7fbfd78041c30558f756.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-13T16:08:14,245 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=1a610c2403a0b491d368473a6e1fb049, regionState=OPEN, openSeqNum=2, regionLocation=f8f85b0bb1d8,38187,1731514088725 2024-11-13T16:08:14,245 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 78285ca993cc7fbfd78041c30558f756 2024-11-13T16:08:14,245 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731514091776.78285ca993cc7fbfd78041c30558f756.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:08:14,245 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for 78285ca993cc7fbfd78041c30558f756 2024-11-13T16:08:14,245 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for 78285ca993cc7fbfd78041c30558f756 2024-11-13T16:08:14,249 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a610c2403a0b491d368473a6e1fb049, server=f8f85b0bb1d8,38187,1731514088725 because future has completed 2024-11-13T16:08:14,249 INFO [StoreOpener-78285ca993cc7fbfd78041c30558f756-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 78285ca993cc7fbfd78041c30558f756 2024-11-13T16:08:14,252 INFO [StoreOpener-78285ca993cc7fbfd78041c30558f756-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 78285ca993cc7fbfd78041c30558f756 columnFamilyName colfamily11 2024-11-13T16:08:14,253 DEBUG [StoreOpener-78285ca993cc7fbfd78041c30558f756-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:14,253 INFO [StoreOpener-78285ca993cc7fbfd78041c30558f756-1 {}] regionserver.HStore(327): Store=78285ca993cc7fbfd78041c30558f756/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:14,254 INFO [StoreOpener-78285ca993cc7fbfd78041c30558f756-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 78285ca993cc7fbfd78041c30558f756 2024-11-13T16:08:14,255 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=12 2024-11-13T16:08:14,256 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 1a610c2403a0b491d368473a6e1fb049, server=f8f85b0bb1d8,38187,1731514088725 in 297 msec 2024-11-13T16:08:14,256 INFO [StoreOpener-78285ca993cc7fbfd78041c30558f756-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 78285ca993cc7fbfd78041c30558f756 columnFamilyName colfamily21 2024-11-13T16:08:14,256 DEBUG [StoreOpener-78285ca993cc7fbfd78041c30558f756-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:14,257 INFO [StoreOpener-78285ca993cc7fbfd78041c30558f756-1 {}] regionserver.HStore(327): Store=78285ca993cc7fbfd78041c30558f756/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:14,257 INFO [StoreOpener-78285ca993cc7fbfd78041c30558f756-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 78285ca993cc7fbfd78041c30558f756 2024-11-13T16:08:14,260 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1a610c2403a0b491d368473a6e1fb049, ASSIGN in 1.5720 sec 2024-11-13T16:08:14,261 INFO [StoreOpener-78285ca993cc7fbfd78041c30558f756-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 78285ca993cc7fbfd78041c30558f756 columnFamilyName colfamily31 2024-11-13T16:08:14,261 DEBUG [StoreOpener-78285ca993cc7fbfd78041c30558f756-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:08:14,262 INFO [StoreOpener-78285ca993cc7fbfd78041c30558f756-1 {}] regionserver.HStore(327): Store=78285ca993cc7fbfd78041c30558f756/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:08:14,262 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for 78285ca993cc7fbfd78041c30558f756 2024-11-13T16:08:14,263 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/78285ca993cc7fbfd78041c30558f756 2024-11-13T16:08:14,264 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/78285ca993cc7fbfd78041c30558f756 2024-11-13T16:08:14,265 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for 78285ca993cc7fbfd78041c30558f756 2024-11-13T16:08:14,265 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for 78285ca993cc7fbfd78041c30558f756 2024-11-13T16:08:14,265 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T16:08:14,266 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for 78285ca993cc7fbfd78041c30558f756 2024-11-13T16:08:14,269 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/78285ca993cc7fbfd78041c30558f756/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T16:08:14,269 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened 78285ca993cc7fbfd78041c30558f756; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75032605, jitterRate=0.11807294189929962}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T16:08:14,270 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 78285ca993cc7fbfd78041c30558f756 2024-11-13T16:08:14,270 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for 78285ca993cc7fbfd78041c30558f756: Running coprocessor pre-open hook at 1731514094245Writing region info on filesystem at 1731514094245Initializing all the Stores at 1731514094249 (+4 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514094249Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514094249Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514094249Cleaning up temporary data from old regions at 1731514094265 (+16 ms)Running coprocessor post-open hooks at 1731514094270 (+5 ms)Region opened successfully at 1731514094270 2024-11-13T16:08:14,271 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731514091776.78285ca993cc7fbfd78041c30558f756., pid=53, masterSystemTime=1731514094081 2024-11-13T16:08:14,274 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731514091776.78285ca993cc7fbfd78041c30558f756. 2024-11-13T16:08:14,274 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731514091776.78285ca993cc7fbfd78041c30558f756. 2024-11-13T16:08:14,275 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=78285ca993cc7fbfd78041c30558f756, regionState=OPEN, openSeqNum=2, regionLocation=f8f85b0bb1d8,38187,1731514088725 2024-11-13T16:08:14,278 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 78285ca993cc7fbfd78041c30558f756, server=f8f85b0bb1d8,38187,1731514088725 because future has completed 2024-11-13T16:08:14,293 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=16 2024-11-13T16:08:14,293 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 78285ca993cc7fbfd78041c30558f756, server=f8f85b0bb1d8,38187,1731514088725 in 355 msec 2024-11-13T16:08:14,306 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=4 2024-11-13T16:08:14,306 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=78285ca993cc7fbfd78041c30558f756, ASSIGN in 1.6100 sec 2024-11-13T16:08:14,309 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T16:08:14,310 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731514094309"}]},"ts":"1731514094309"} 2024-11-13T16:08:14,314 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, state=ENABLED in hbase:meta 2024-11-13T16:08:14,315 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T16:08:14,323 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true in 2.5290 sec 2024-11-13T16:08:15,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T16:08:15,999 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true get assigned. Timeout = 60000ms 2024-11-13T16:08:15,999 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true completed 2024-11-13T16:08:16,001 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T16:08:16,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43467 {}] regionserver.StoreScanner(1138): Switch to stream read (scanned=32802 bytes) of info 2024-11-13T16:08:16,024 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true assigned to meta. Checking AM states. 2024-11-13T16:08:16,025 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T16:08:16,026 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true assigned. 2024-11-13T16:08:16,076 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aaa', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731514091776.4aaf5667f132c289386d1b3b0c0ac57f., hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=2] 2024-11-13T16:08:16,080 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aac', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731514091776.4aaf5667f132c289386d1b3b0c0ac57f., hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=2] 2024-11-13T16:08:16,084 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aad', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731514091776.4aaf5667f132c289386d1b3b0c0ac57f., hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=2] 2024-11-13T16:08:16,085 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aab', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731514091776.4aaf5667f132c289386d1b3b0c0ac57f., hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=2] 2024-11-13T16:08:16,087 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aae', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731514091776.4aaf5667f132c289386d1b3b0c0ac57f., hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=2] 2024-11-13T16:08:16,089 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aaf', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731514091776.4aaf5667f132c289386d1b3b0c0ac57f., hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=2] 2024-11-13T16:08:16,090 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aag', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731514091776.4aaf5667f132c289386d1b3b0c0ac57f., hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=2] 2024-11-13T16:08:16,092 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aah', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731514091776.4aaf5667f132c289386d1b3b0c0ac57f., hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=2] 2024-11-13T16:08:16,166 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='bbb', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731514091776.8ba796d80fc3b65f0948c95ccbdad39d., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2] 2024-11-13T16:08:16,171 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='bbd', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731514091776.8ba796d80fc3b65f0948c95ccbdad39d., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2] 2024-11-13T16:08:16,178 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='bbc', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731514091776.8ba796d80fc3b65f0948c95ccbdad39d., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2] 2024-11-13T16:08:16,180 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='bbf', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731514091776.8ba796d80fc3b65f0948c95ccbdad39d., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2] 2024-11-13T16:08:16,181 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='bbe', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731514091776.8ba796d80fc3b65f0948c95ccbdad39d., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2] 2024-11-13T16:08:16,246 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ccc', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731514091776.93f8ed38b68bd013eb0cf0e7615013ae., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:16,248 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ccd', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731514091776.93f8ed38b68bd013eb0cf0e7615013ae., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:16,250 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='cce', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731514091776.93f8ed38b68bd013eb0cf0e7615013ae., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:16,252 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ccf', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731514091776.93f8ed38b68bd013eb0cf0e7615013ae., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:16,254 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ccg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731514091776.93f8ed38b68bd013eb0cf0e7615013ae., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:16,265 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='cci', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731514091776.93f8ed38b68bd013eb0cf0e7615013ae., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:16,267 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='cch', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731514091776.93f8ed38b68bd013eb0cf0e7615013ae., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:16,271 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ccj', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731514091776.93f8ed38b68bd013eb0cf0e7615013ae., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:16,337 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddd', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731514091776.e72e57b96ff9bc0bbd862ce564d79c42., hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=2] 2024-11-13T16:08:16,343 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='dde', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731514091776.e72e57b96ff9bc0bbd862ce564d79c42., hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=2] 2024-11-13T16:08:16,345 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddf', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731514091776.e72e57b96ff9bc0bbd862ce564d79c42., hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=2] 2024-11-13T16:08:16,346 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731514091776.e72e57b96ff9bc0bbd862ce564d79c42., hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=2] 2024-11-13T16:08:16,357 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddh', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731514091776.e72e57b96ff9bc0bbd862ce564d79c42., hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=2] 2024-11-13T16:08:16,415 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eee', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731514091776.34f3d3b6a6dae1f4077567bed0bd8e75., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:16,432 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eeg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731514091776.34f3d3b6a6dae1f4077567bed0bd8e75., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:16,438 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eeh', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731514091776.34f3d3b6a6dae1f4077567bed0bd8e75., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:16,441 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eef', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731514091776.34f3d3b6a6dae1f4077567bed0bd8e75., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:16,500 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='fff', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731514091776.3945ce860c8642051e17517944a83136., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2] 2024-11-13T16:08:16,504 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ffg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731514091776.3945ce860c8642051e17517944a83136., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2] 2024-11-13T16:08:16,506 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ffh', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731514091776.3945ce860c8642051e17517944a83136., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2] 2024-11-13T16:08:16,509 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ffi', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731514091776.3945ce860c8642051e17517944a83136., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2] 2024-11-13T16:08:16,512 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ffk', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731514091776.3945ce860c8642051e17517944a83136., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2] 2024-11-13T16:08:16,516 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ffj', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731514091776.3945ce860c8642051e17517944a83136., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2] 2024-11-13T16:08:16,518 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ffl', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731514091776.3945ce860c8642051e17517944a83136., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2] 2024-11-13T16:08:16,520 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ffm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731514091776.3945ce860c8642051e17517944a83136., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2] 2024-11-13T16:08:16,588 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggh', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731514091776.1fcf78578ccab75f1705f3c2d7bcdb14., hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=2] 2024-11-13T16:08:16,593 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggi', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731514091776.1fcf78578ccab75f1705f3c2d7bcdb14., hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=2] 2024-11-13T16:08:16,596 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731514091776.1fcf78578ccab75f1705f3c2d7bcdb14., hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=2] 2024-11-13T16:08:16,598 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggj', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731514091776.1fcf78578ccab75f1705f3c2d7bcdb14., hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=2] 2024-11-13T16:08:16,602 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggk', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731514091776.1fcf78578ccab75f1705f3c2d7bcdb14., hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=2] 2024-11-13T16:08:16,605 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggl', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731514091776.1fcf78578ccab75f1705f3c2d7bcdb14., hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=2] 2024-11-13T16:08:16,607 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731514091776.1fcf78578ccab75f1705f3c2d7bcdb14., hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=2] 2024-11-13T16:08:16,610 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggn', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731514091776.1fcf78578ccab75f1705f3c2d7bcdb14., hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=2] 2024-11-13T16:08:16,649 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhh', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731514091776.1a610c2403a0b491d368473a6e1fb049., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:16,652 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhi', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731514091776.1a610c2403a0b491d368473a6e1fb049., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:16,656 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhj', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731514091776.1a610c2403a0b491d368473a6e1fb049., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:16,658 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhk', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731514091776.1a610c2403a0b491d368473a6e1fb049., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:16,660 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731514091776.1a610c2403a0b491d368473a6e1fb049., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:16,662 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhl', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731514091776.1a610c2403a0b491d368473a6e1fb049., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:16,664 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhn', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731514091776.1a610c2403a0b491d368473a6e1fb049., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:16,672 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hho', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731514091776.1a610c2403a0b491d368473a6e1fb049., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:16,723 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iii', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731514091776.e8725a794e6d251e1bf6e3502baa7700., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2] 2024-11-13T16:08:16,726 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iij', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731514091776.e8725a794e6d251e1bf6e3502baa7700., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2] 2024-11-13T16:08:16,728 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iik', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731514091776.e8725a794e6d251e1bf6e3502baa7700., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2] 2024-11-13T16:08:16,730 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iil', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731514091776.e8725a794e6d251e1bf6e3502baa7700., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2] 2024-11-13T16:08:16,733 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iim', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731514091776.e8725a794e6d251e1bf6e3502baa7700., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2] 2024-11-13T16:08:16,747 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T16:08:16,805 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjj', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731514091776.fa6636be3559bc84793534420cd4f207., hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=2] 2024-11-13T16:08:16,807 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjk', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731514091776.fa6636be3559bc84793534420cd4f207., hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=2] 2024-11-13T16:08:16,811 DEBUG [Time-limited test {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjl', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731514091776.fa6636be3559bc84793534420cd4f207., hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=2] 2024-11-13T16:08:16,891 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true' 2024-11-13T16:08:16,896 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-13T16:08:16,907 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkl', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731514091776.05d65f4a6bfecb3d530d15832c218252., hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=2] 2024-11-13T16:08:16,911 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731514091776.05d65f4a6bfecb3d530d15832c218252., hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=2] 2024-11-13T16:08:16,914 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkk', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731514091776.05d65f4a6bfecb3d530d15832c218252., hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=2] 2024-11-13T16:08:16,930 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkn', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731514091776.05d65f4a6bfecb3d530d15832c218252., hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=2] 2024-11-13T16:08:16,933 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731514091776.05d65f4a6bfecb3d530d15832c218252., hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=2] 2024-11-13T16:08:16,944 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kko', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731514091776.05d65f4a6bfecb3d530d15832c218252., hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=2] 2024-11-13T16:08:17,000 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='lll', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731514091776.78285ca993cc7fbfd78041c30558f756., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:17,004 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='llm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731514091776.78285ca993cc7fbfd78041c30558f756., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:17,007 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='llo', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731514091776.78285ca993cc7fbfd78041c30558f756., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:17,009 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='lln', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731514091776.78285ca993cc7fbfd78041c30558f756., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:17,012 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='llp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731514091776.78285ca993cc7fbfd78041c30558f756., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:17,014 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='llq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731514091776.78285ca993cc7fbfd78041c30558f756., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:17,024 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='llr', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731514091776.78285ca993cc7fbfd78041c30558f756., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:17,028 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='lls', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731514091776.78285ca993cc7fbfd78041c30558f756., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:17,083 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731514091776.aa124b802c708ca1f715cb1f8c9c4eaa., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2] 2024-11-13T16:08:17,088 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmo', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731514091776.aa124b802c708ca1f715cb1f8c9c4eaa., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2] 2024-11-13T16:08:17,103 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731514091776.aa124b802c708ca1f715cb1f8c9c4eaa., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2] 2024-11-13T16:08:17,107 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731514091776.aa124b802c708ca1f715cb1f8c9c4eaa., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2] 2024-11-13T16:08:17,114 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmr', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731514091776.aa124b802c708ca1f715cb1f8c9c4eaa., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2] 2024-11-13T16:08:17,124 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mms', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731514091776.aa124b802c708ca1f715cb1f8c9c4eaa., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2] 2024-11-13T16:08:17,127 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmt', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731514091776.aa124b802c708ca1f715cb1f8c9c4eaa., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2] 2024-11-13T16:08:17,130 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmn', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731514091776.aa124b802c708ca1f715cb1f8c9c4eaa., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2] 2024-11-13T16:08:17,161 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nnn', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731514091776.3647edbd8aef23dda3887b61277da1d3., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:17,170 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nno', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731514091776.3647edbd8aef23dda3887b61277da1d3., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:17,172 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nnp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731514091776.3647edbd8aef23dda3887b61277da1d3., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:17,174 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nnq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731514091776.3647edbd8aef23dda3887b61277da1d3., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:17,180 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nnr', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731514091776.3647edbd8aef23dda3887b61277da1d3., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:17,182 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nns', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731514091776.3647edbd8aef23dda3887b61277da1d3., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:17,184 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nnt', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731514091776.3647edbd8aef23dda3887b61277da1d3., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:17,200 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nnu', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731514091776.3647edbd8aef23dda3887b61277da1d3., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:17,261 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ooo', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731514091776.29df624bbcb878edce7fcdf0abceea88., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2] 2024-11-13T16:08:17,267 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='oop', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731514091776.29df624bbcb878edce7fcdf0abceea88., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2] 2024-11-13T16:08:17,269 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ooq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731514091776.29df624bbcb878edce7fcdf0abceea88., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2] 2024-11-13T16:08:17,273 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='oor', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731514091776.29df624bbcb878edce7fcdf0abceea88., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2] 2024-11-13T16:08:17,280 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='oot', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731514091776.29df624bbcb878edce7fcdf0abceea88., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2] 2024-11-13T16:08:17,282 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='oou', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731514091776.29df624bbcb878edce7fcdf0abceea88., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2] 2024-11-13T16:08:17,284 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='oos', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731514091776.29df624bbcb878edce7fcdf0abceea88., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2] 2024-11-13T16:08:17,335 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731514091776.fe97fe0e9b37811ba99ca8100b461589., hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=2] 2024-11-13T16:08:17,338 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731514091776.fe97fe0e9b37811ba99ca8100b461589., hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=2] 2024-11-13T16:08:17,341 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='pps', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731514091776.fe97fe0e9b37811ba99ca8100b461589., hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=2] 2024-11-13T16:08:17,343 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppr', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731514091776.fe97fe0e9b37811ba99ca8100b461589., hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=2] 2024-11-13T16:08:17,352 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppt', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731514091776.fe97fe0e9b37811ba99ca8100b461589., hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=2] 2024-11-13T16:08:17,355 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppu', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731514091776.fe97fe0e9b37811ba99ca8100b461589., hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=2] 2024-11-13T16:08:17,357 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731514091776.fe97fe0e9b37811ba99ca8100b461589., hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=2] 2024-11-13T16:08:17,404 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqr', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731514091776.7c34e9ded13254b91545186c85befbf8., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:17,408 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731514091776.7c34e9ded13254b91545186c85befbf8., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:17,418 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqt', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731514091776.7c34e9ded13254b91545186c85befbf8., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:17,420 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqs', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731514091776.7c34e9ded13254b91545186c85befbf8., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:17,421 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqu', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731514091776.7c34e9ded13254b91545186c85befbf8., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:17,423 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731514091776.7c34e9ded13254b91545186c85befbf8., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:17,424 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqw', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731514091776.7c34e9ded13254b91545186c85befbf8., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:17,426 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqx', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731514091776.7c34e9ded13254b91545186c85befbf8., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:17,470 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rrr', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731514091776.f8f8b8e4ae8470fd8a1d2bf0684947ef., hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=2] 2024-11-13T16:08:17,473 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rrs', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731514091776.f8f8b8e4ae8470fd8a1d2bf0684947ef., hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=2] 2024-11-13T16:08:17,476 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rrt', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731514091776.f8f8b8e4ae8470fd8a1d2bf0684947ef., hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=2] 2024-11-13T16:08:17,478 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rru', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731514091776.f8f8b8e4ae8470fd8a1d2bf0684947ef., hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=2] 2024-11-13T16:08:17,558 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='sss', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731514091776.fec5f21b48c36bf01685ec2ee58f0490., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2] 2024-11-13T16:08:17,561 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='sst', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731514091776.fec5f21b48c36bf01685ec2ee58f0490., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2] 2024-11-13T16:08:17,564 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ssu', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731514091776.fec5f21b48c36bf01685ec2ee58f0490., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2] 2024-11-13T16:08:17,566 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ssv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731514091776.fec5f21b48c36bf01685ec2ee58f0490., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2] 2024-11-13T16:08:17,569 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ssw', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731514091776.fec5f21b48c36bf01685ec2ee58f0490., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2] 2024-11-13T16:08:17,588 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ssx', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731514091776.fec5f21b48c36bf01685ec2ee58f0490., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2] 2024-11-13T16:08:17,590 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ssy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731514091776.fec5f21b48c36bf01685ec2ee58f0490., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2] 2024-11-13T16:08:17,592 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ssz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731514091776.fec5f21b48c36bf01685ec2ee58f0490., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2] 2024-11-13T16:08:17,620 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ttt', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731514091776.97998e7179d85b23a3f5928cd6511629., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:17,623 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ttu', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731514091776.97998e7179d85b23a3f5928cd6511629., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:17,625 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ttv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731514091776.97998e7179d85b23a3f5928cd6511629., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:17,638 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ttw', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731514091776.97998e7179d85b23a3f5928cd6511629., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:17,640 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ttx', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731514091776.97998e7179d85b23a3f5928cd6511629., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:17,692 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uuu', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731514091776.519ece296a21b5ddccc38a7c928780bb., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2] 2024-11-13T16:08:17,700 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uuv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731514091776.519ece296a21b5ddccc38a7c928780bb., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2] 2024-11-13T16:08:17,702 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uux', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731514091776.519ece296a21b5ddccc38a7c928780bb., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2] 2024-11-13T16:08:17,704 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uuw', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731514091776.519ece296a21b5ddccc38a7c928780bb., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2] 2024-11-13T16:08:17,768 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731514091776.c4a87adf770b37d40ab6691e377ad1b5., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:17,778 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvw', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731514091776.c4a87adf770b37d40ab6691e377ad1b5., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:17,782 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731514091776.c4a87adf770b37d40ab6691e377ad1b5., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:17,784 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvx', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731514091776.c4a87adf770b37d40ab6691e377ad1b5., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:17,786 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731514091776.c4a87adf770b37d40ab6691e377ad1b5., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:17,789 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vwa', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731514091776.c4a87adf770b37d40ab6691e377ad1b5., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:17,831 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='www', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731514091776.bf1f9f9bc1bc9ffce566d41f8ae3619a., hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=2] 2024-11-13T16:08:17,834 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='wwx', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731514091776.bf1f9f9bc1bc9ffce566d41f8ae3619a., hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=2] 2024-11-13T16:08:17,836 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='wwy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731514091776.bf1f9f9bc1bc9ffce566d41f8ae3619a., hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=2] 2024-11-13T16:08:17,840 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='wwz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731514091776.bf1f9f9bc1bc9ffce566d41f8ae3619a., hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=2] 2024-11-13T16:08:17,882 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xxx', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731514091776.4a146f98682f25e8cddcae5dbf318603., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:17,885 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xxy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731514091776.4a146f98682f25e8cddcae5dbf318603., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:17,887 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xxz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731514091776.4a146f98682f25e8cddcae5dbf318603., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:17,889 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xya', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731514091776.4a146f98682f25e8cddcae5dbf318603., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:17,891 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xyb', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731514091776.4a146f98682f25e8cddcae5dbf318603., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:17,893 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xyc', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731514091776.4a146f98682f25e8cddcae5dbf318603., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=2] 2024-11-13T16:08:17,938 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yyy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731514091776.6be98d350311c17b11fdb9e9a1238fca., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2] 2024-11-13T16:08:17,940 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yyz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731514091776.6be98d350311c17b11fdb9e9a1238fca., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2] 2024-11-13T16:08:17,945 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yza', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731514091776.6be98d350311c17b11fdb9e9a1238fca., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2] 2024-11-13T16:08:17,947 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yzb', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731514091776.6be98d350311c17b11fdb9e9a1238fca., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2] 2024-11-13T16:08:17,949 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yzc', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731514091776.6be98d350311c17b11fdb9e9a1238fca., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2] 2024-11-13T16:08:17,951 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yzd', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731514091776.6be98d350311c17b11fdb9e9a1238fca., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2] 2024-11-13T16:08:17,953 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yze', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731514091776.6be98d350311c17b11fdb9e9a1238fca., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2] 2024-11-13T16:08:17,993 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-13T16:08:17,994 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='zzz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731514091776.72d273770314688e78624993de4f3398., hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=2] 2024-11-13T16:08:17,994 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-13T16:08:17,997 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-13T16:08:17,997 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true Metrics about Tables on a single HBase RegionServer 2024-11-13T16:08:17,999 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T16:08:17,999 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-13T16:08:17,999 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-13T16:08:18,000 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-13T16:08:18,048 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T16:08:18,087 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T16:08:18,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43467 {}] regionserver.RSRpcServices(2610): Large batch operation detected (greater than 5000) (HBASE-18023). Requested Number of Rows: 5624 Client: jenkins//172.17.0.3 first region in multi=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731514091776.8ba796d80fc3b65f0948c95ccbdad39d. 2024-11-13T16:08:18,092 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57930, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T16:08:18,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41333 {}] regionserver.RSRpcServices(2610): Large batch operation detected (greater than 5000) (HBASE-18023). Requested Number of Rows: 5625 Client: jenkins//172.17.0.3 first region in multi=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731514091776.4aaf5667f132c289386d1b3b0c0ac57f. 2024-11-13T16:08:18,119 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51662, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T16:08:18,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38187 {}] regionserver.RSRpcServices(2610): Large batch operation detected (greater than 5000) (HBASE-18023). Requested Number of Rows: 6327 Client: jenkins//172.17.0.3 first region in multi=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731514091776.93f8ed38b68bd013eb0cf0e7615013ae. 2024-11-13T16:08:19,384 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,, stopping at row=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true ,, for max=2147483647 with caching=100 2024-11-13T16:08:19,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43467 {}] regionserver.StoreScanner(1138): Switch to stream read (scanned=32802 bytes) of info 2024-11-13T16:08:19,653 WARN [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: true 2024-11-13T16:08:19,654 INFO [Time-limited test {}] assignment.AssignmentTestingUtil(82): KILL REGION SERVER f8f85b0bb1d8,43467,1731514088560 2024-11-13T16:08:19,654 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing f8f85b0bb1d8,43467,1731514088560 2024-11-13T16:08:19,655 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server f8f85b0bb1d8,43467,1731514088560: Simulated kill ***** 2024-11-13T16:08:19,655 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-13T16:08:19,658 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-13T16:08:19,659 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-13T16:08:19,663 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-13T16:08:19,665 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-13T16:08:19,688 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 646140416 }, "NonHeapMemoryUsage": { "committed": 159383552, "init": 7667712, "max": -1, "used": 156541472 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "f8f85b0bb1d8", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 3, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 61484, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 133, "ProcessCallTime_min": 1, "ProcessCallTime_max": 24, "ProcessCallTime_mean": 3, "ProcessCallTime_25th_percentile": 6, "ProcessCallTime_median": 12, "ProcessCallTime_75th_percentile": 18, "ProcessCallTime_90th_percentile": 21, "ProcessCallTime_95th_percentile": 22, "ProcessCallTime_98th_percentile": 23, "ProcessCallTime_99th_percentile": 23, "ProcessCallTime_99.9th_percentile": 23, "ProcessCallTime_TimeRangeCount_0-1": 133, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 133, "QueueCallTime_min": 0, "QueueCallTime_max": 49, "QueueCallTime_mean": 3, "QueueCallTime_25th_percentile": 12, "QueueCallTime_median": 24, "QueueCallTime_75th_percentile": 36, "QueueCallTime_90th_percentile": 44, "QueueCallTime_95th_percentile": 46, "QueueCallTime_98th_percentile": 48, "QueueCallTime_99th_percentile": 48, "QueueCallTime_99.9th_percentile": 48, "QueueCallTime_TimeRangeCount_0-1": 133, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 133, "TotalCallTime_min": 1, "TotalCallTime_max": 59, "TotalCallTime_mean": 7, "TotalCallTime_25th_percentile": 15, "TotalCallTime_median": 30, "TotalCallTime_75th_percentile": 44, "TotalCallTime_90th_percentile": 53, "TotalCallTime_95th_percentile": 56, "TotalCallTime_98th_percentile": 57, "TotalCallTime_99th_percentile": 58, "TotalCallTime_99.9th_percentile": 58, "TotalCallTime_TimeRangeCount_0-1": 133, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 133, "ResponseSize_min": 0, "ResponseSize_max": 23824, "ResponseSize_mean": 3494, "ResponseSize_25th_percentile": 5956, "ResponseSize_median": 11912, "ResponseSize_75th_percentile": 17868, "ResponseSize_90th_percentile": 21441, "ResponseSize_95th_percentile": 22632, "ResponseSize_98th_percentile": 23347, "ResponseSize_99th_percentile": 23585, "ResponseSize_99.9th_percentile": 23800, "ResponseSize_SizeRangeCount_0-10": 133, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 133, "RequestSize_min": 45, "RequestSize_max": 14907, "RequestSize_mean": 443, "RequestSize_25th_percentile": 3760, "RequestSize_median": 7476, "RequestSize_75th_percentile": 11191, "RequestSize_90th_percentile": 13420, "RequestSize_95th_percentile": 14163, "RequestSize_98th_percentile": 14609, "RequestSize_99th_percentile": 14758, "RequestSize_99.9th_percentile": 14892, "RequestSize_SizeRangeCount_0-10": 133, "sentBytes": 398998 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "f8f85b0bb1d8", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:60455", "tag.serverName": "f8f85b0bb1d8,43467,1731514088560", "tag.clusterId": "5d0c4b15-0fa1-4b67-8c04-53b67980164f", "tag.Context": "regionserver", "tag.Hostname": "f8f85b0bb1d8", "regionCount": 9, "storeCount": 28, "hlogFileCount": 2, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 50772, "memStoreHeapSize": 77808, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1731514088560, "averageRegionSize": 5641, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 0, "localBytesRead": 0, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 1.4, "writeRequestRatePerSecond": 16.400000000000002, "ByteBuffAllocatorHeapAllocationBytes": 0, "ByteBuffAllocatorPoolAllocationBytes": 4459520, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 4, "activeScanners": 0, "totalRequestCount": 126, "totalRowActionRequestCount": 89, "readRequestCount": 7, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 82, "rpcGetRequestCount": 1, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 68, "rpcMultiRequestCount": 1, "rpcMutateRequestCount": 56, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 68, "ScanTime_min": 0, "ScanTime_max": 7, "ScanTime_mean": 1, "ScanTime_25th_percentile": 1, "ScanTime_median": 3, "ScanTime_75th_percentile": 5, "ScanTime_90th_percentile": 6, "ScanTime_95th_percentile": 6, "ScanTime_98th_percentile": 6, "ScanTime_99th_percentile": 6, "ScanTime_99.9th_percentile": 6, "ScanTime_TimeRangeCount_0-1": 68, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 56, "Put_min": 1, "Put_max": 14, "Put_mean": 4, "Put_25th_percentile": 4, "Put_median": 7, "Put_75th_percentile": 10, "Put_90th_percentile": 12, "Put_95th_percentile": 13, "Put_98th_percentile": 13, "Put_99th_percentile": 13, "Put_99.9th_percentile": 13, "Put_TimeRangeCount_0-1": 56, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 1, "PutBatch_min": 15, "PutBatch_max": 15, "PutBatch_mean": 15, "PutBatch_25th_percentile": 15, "PutBatch_median": 15, "PutBatch_75th_percentile": 15, "PutBatch_90th_percentile": 15, "PutBatch_95th_percentile": 15, "PutBatch_98th_percentile": 15, "PutBatch_99th_percentile": 15, "PutBatch_99.9th_percentile": 15, "PutBatch_TimeRangeCount_0-1": 1, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 1, "Get_min": 2, "Get_max": 2, "Get_mean": 2, "Get_25th_percentile": 2, "Get_median": 2, "Get_75th_percentile": 2, "Get_90th_percentile": 2, "Get_95th_percentile": 2, "Get_98th_percentile": 2, "Get_99th_percentile": 2, "Get_99.9th_percentile": 2, "Get_TimeRangeCount_0-1": 1, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 68, "ScanSize_min": 0, "ScanSize_max": 23728, "ScanSize_mean": 6788, "ScanSize_25th_percentile": 5932, "ScanSize_median": 11864, "ScanSize_75th_percentile": 17796, "ScanSize_90th_percentile": 21355, "ScanSize_95th_percentile": 22541, "ScanSize_98th_percentile": 23253, "ScanSize_99th_percentile": 23490, "ScanSize_99.9th_percentile": 23704, "ScanSize_SizeRangeCount_0-10": 68, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-13T16:08:19,692 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42949 {}] master.MasterRpcServices(700): f8f85b0bb1d8,43467,1731514088560 reported a fatal error: ***** ABORTING region server f8f85b0bb1d8,43467,1731514088560: Simulated kill ***** 2024-11-13T16:08:19,700 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f8f85b0bb1d8,43467,1731514088560' ***** 2024-11-13T16:08:19,700 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-13T16:08:19,701 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T16:08:19,701 INFO [RS:0;f8f85b0bb1d8:43467 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T16:08:19,702 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T16:08:19,702 INFO [RS:0;f8f85b0bb1d8:43467 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-13T16:08:19,702 INFO [RS:0;f8f85b0bb1d8:43467 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-13T16:08:19,702 INFO [RS:0;f8f85b0bb1d8:43467 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T16:08:19,703 INFO [RS:0;f8f85b0bb1d8:43467 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;f8f85b0bb1d8:43467. 2024-11-13T16:08:19,703 DEBUG [RS:0;f8f85b0bb1d8:43467 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T16:08:19,711 DEBUG [RS:0;f8f85b0bb1d8:43467 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T16:08:19,731 INFO [regionserver/f8f85b0bb1d8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T16:08:19,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741836_1012 (size=54214) 2024-11-13T16:08:19,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741836_1012 (size=54214) 2024-11-13T16:08:19,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741836_1012 (size=54214) 2024-11-13T16:08:19,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741834_1010 (size=297717) 2024-11-13T16:08:19,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741834_1010 (size=297717) 2024-11-13T16:08:19,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741834_1010 (size=297717) 2024-11-13T16:08:19,765 DEBUG [RS:0;f8f85b0bb1d8:43467 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T16:08:19,765 INFO [RS:0;f8f85b0bb1d8:43467 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T16:08:19,765 INFO [RS:0;f8f85b0bb1d8:43467 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:43467 2024-11-13T16:08:19,773 INFO [regionserver/f8f85b0bb1d8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-13T16:08:19,773 INFO [regionserver/f8f85b0bb1d8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-13T16:08:19,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42949-0x1013621fa650000, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T16:08:19,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43467-0x1013621fa650001, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f8f85b0bb1d8,43467,1731514088560 2024-11-13T16:08:19,784 INFO [RS:0;f8f85b0bb1d8:43467 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T16:08:19,787 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f8f85b0bb1d8,43467,1731514088560] 2024-11-13T16:08:19,803 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f8f85b0bb1d8,43467,1731514088560 already deleted, retry=false 2024-11-13T16:08:19,803 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of f8f85b0bb1d8,43467,1731514088560 on f8f85b0bb1d8,42949,1731514087490 2024-11-13T16:08:19,811 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure f8f85b0bb1d8,43467,1731514088560, splitWal=true, meta=true 2024-11-13T16:08:19,820 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=57 for f8f85b0bb1d8,43467,1731514088560 (carryingMeta=true) f8f85b0bb1d8,43467,1731514088560/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@15fece12[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-13T16:08:19,823 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure f8f85b0bb1d8,43467,1731514088560, splitWal=true, meta=true 2024-11-13T16:08:19,823 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-13T16:08:19,824 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure f8f85b0bb1d8,43467,1731514088560, splitWal=true, meta=true 2024-11-13T16:08:19,824 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T16:08:19,824 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure f8f85b0bb1d8,43467,1731514088560, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T16:08:19,829 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T16:08:19,831 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 0 exec state=pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure f8f85b0bb1d8,43467,1731514088560, splitWal=true, meta=true 2024-11-13T16:08:19,832 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-13T16:08:19,832 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T16:08:19,833 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T16:08:19,833 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T16:08:19,834 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-13T16:08:19,834 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-13T16:08:19,834 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-13T16:08:19,835 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T16:08:19,836 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-13T16:08:19,842 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-13T16:08:19,843 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T16:08:19,848 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-13T16:08:19,848 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure f8f85b0bb1d8,43467,1731514088560, splitWal=true, meta=true 2024-11-13T16:08:19,849 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure f8f85b0bb1d8,43467,1731514088560, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-13T16:08:19,849 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 13 msec 2024-11-13T16:08:19,858 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=f8f85b0bb1d8,43467,1731514088560, regionLocation=f8f85b0bb1d8,43467,1731514088560, openSeqNum=2 2024-11-13T16:08:19,858 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=f8f85b0bb1d8,43467,1731514088560, table=hbase:meta, region=1588230740 2024-11-13T16:08:19,858 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f8f85b0bb1d8,43467,1731514088560, state=OPEN 2024-11-13T16:08:19,882 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41333-0x1013621fa650003, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T16:08:19,882 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:08:19,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42949-0x1013621fa650000, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T16:08:19,883 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:08:19,887 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38187-0x1013621fa650002, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T16:08:19,887 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:08:19,891 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731514129891 2024-11-13T16:08:19,891 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-13T16:08:19,891 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure f8f85b0bb1d8,43467,1731514088560, splitWal=true, meta=true 2024-11-13T16:08:19,891 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-13T16:08:19,894 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure f8f85b0bb1d8,43467,1731514088560, splitWal=true, meta=true, isMeta: true 2024-11-13T16:08:19,894 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server f8f85b0bb1d8:43467 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:08:19,896 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address f8f85b0bb1d8:43467 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:08:19,897 INFO [RS:0;f8f85b0bb1d8:43467 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T16:08:19,897 INFO [RS:0;f8f85b0bb1d8:43467 {}] regionserver.HRegionServer(1031): Exiting; stopping=f8f85b0bb1d8,43467,1731514088560; zookeeper connection closed. 2024-11-13T16:08:19,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43467-0x1013621fa650001, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T16:08:19,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43467-0x1013621fa650001, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T16:08:19,899 DEBUG [Time-limited test {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1, error=java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:08:19,900 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/WALs/f8f85b0bb1d8,43467,1731514088560-splitting 2024-11-13T16:08:19,900 DEBUG [Time-limited test {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T16:08:19,900 DEBUG [Time-limited test {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 from cache 2024-11-13T16:08:19,901 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5f5ab1bc {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5f5ab1bc 2024-11-13T16:08:19,903 INFO [PEWorker-1 {}] master.SplitWALManager(105): f8f85b0bb1d8,43467,1731514088560 WAL count=1, meta=true 2024-11-13T16:08:19,906 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure f8f85b0bb1d8%2C43467%2C1731514088560.meta.1731514091182.meta}] 2024-11-13T16:08:19,906 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-13T16:08:19,906 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, hasLock=true; ServerCrashProcedure f8f85b0bb1d8,43467,1731514088560, splitWal=true, meta=true 2024-11-13T16:08:19,906 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T16:08:19,907 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, hasLock=true; ServerCrashProcedure f8f85b0bb1d8,43467,1731514088560, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T16:08:20,007 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T16:08:20,007 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1] 2024-11-13T16:08:20,008 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to f8f85b0bb1d8:43467 this server is in the failed servers list 2024-11-13T16:08:20,008 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=f8f85b0bb1d8:43467 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:43467 2024-11-13T16:08:20,008 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:43467 2024-11-13T16:08:20,008 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 from cache 2024-11-13T16:08:20,216 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T16:08:20,217 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1] 2024-11-13T16:08:20,217 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to f8f85b0bb1d8:43467 this server is in the failed servers list 2024-11-13T16:08:20,218 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=f8f85b0bb1d8:43467 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:43467 2024-11-13T16:08:20,218 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:43467 2024-11-13T16:08:20,218 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 from cache 2024-11-13T16:08:20,526 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T16:08:20,526 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1] 2024-11-13T16:08:20,526 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to f8f85b0bb1d8:43467 this server is in the failed servers list 2024-11-13T16:08:20,527 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=f8f85b0bb1d8:43467 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:43467 2024-11-13T16:08:20,527 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:43467 2024-11-13T16:08:20,527 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 from cache 2024-11-13T16:08:21,036 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T16:08:21,036 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1] 2024-11-13T16:08:21,037 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to f8f85b0bb1d8:43467 this server is in the failed servers list 2024-11-13T16:08:21,037 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=f8f85b0bb1d8:43467 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:43467 2024-11-13T16:08:21,037 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:43467 2024-11-13T16:08:21,037 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 from cache 2024-11-13T16:08:22,056 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T16:08:22,056 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1] 2024-11-13T16:08:22,058 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server f8f85b0bb1d8:43467 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:08:22,062 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to f8f85b0bb1d8,43467,1731514088560 for '' in 1588230740 of hbase:meta failed, tries = 6, maxAttempts = 46, timeout = 60000 ms, time elapsed = 2166 ms java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T16:08:22,063 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1, error=java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:08:22,063 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T16:08:22,063 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 from cache 2024-11-13T16:08:22,064 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address f8f85b0bb1d8:43467 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:08:24,066 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T16:08:24,067 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1] 2024-11-13T16:08:24,068 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server f8f85b0bb1d8:43467 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:08:24,069 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to f8f85b0bb1d8,43467,1731514088560 for '' in 1588230740 of hbase:meta failed, tries = 7, maxAttempts = 46, timeout = 60000 ms, time elapsed = 4177 ms java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T16:08:24,070 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1, error=java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:08:24,070 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T16:08:24,070 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 from cache 2024-11-13T16:08:24,070 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address f8f85b0bb1d8:43467 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:08:28,096 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T16:08:28,096 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1] 2024-11-13T16:08:28,097 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server f8f85b0bb1d8:43467 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:08:28,098 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to f8f85b0bb1d8,43467,1731514088560 for '' in 1588230740 of hbase:meta failed, tries = 8, maxAttempts = 46, timeout = 60000 ms, time elapsed = 8206 ms java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T16:08:28,099 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1, error=java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:08:28,099 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T16:08:28,099 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 from cache 2024-11-13T16:08:28,099 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address f8f85b0bb1d8:43467 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:08:36,530 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T16:08:38,176 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T16:08:38,176 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1] 2024-11-13T16:08:38,177 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server f8f85b0bb1d8:43467 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:08:38,178 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to f8f85b0bb1d8,43467,1731514088560 for '' in 1588230740 of hbase:meta failed, tries = 9, maxAttempts = 46, timeout = 60000 ms, time elapsed = 18286 ms java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T16:08:38,179 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1, error=java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:08:38,179 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T16:08:38,179 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 from cache 2024-11-13T16:08:38,180 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address f8f85b0bb1d8:43467 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:08:44,193 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=f8f85b0bb1d8:42949 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 10 more 2024-11-13T16:08:45,950 WARN [RS:1;f8f85b0bb1d8:38187 {}] monitoring.TaskMonitor(166): Status Processing ServerCrashProcedure of f8f85b0bb1d8,43467,1731514088560: status=status unset, state=RUNNING, startTime=1731514099822, completionTime=-1 appears to have been leaked 2024-11-13T16:08:48,257 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T16:08:48,257 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1] 2024-11-13T16:08:48,258 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server f8f85b0bb1d8:43467 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:08:48,259 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to f8f85b0bb1d8,43467,1731514088560 for '' in 1588230740 of hbase:meta failed, tries = 10, maxAttempts = 46, timeout = 60000 ms, time elapsed = 28367 ms java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T16:08:48,260 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1, error=java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:08:48,260 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T16:08:48,260 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 from cache 2024-11-13T16:08:48,260 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address f8f85b0bb1d8:43467 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:08:52,212 INFO [master/f8f85b0bb1d8:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-13T16:08:52,212 INFO [master/f8f85b0bb1d8:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-13T16:08:58,356 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T16:08:58,356 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1] 2024-11-13T16:08:58,357 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server f8f85b0bb1d8:43467 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:08:58,358 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to f8f85b0bb1d8,43467,1731514088560 for '' in 1588230740 of hbase:meta failed, tries = 11, maxAttempts = 46, timeout = 60000 ms, time elapsed = 38466 ms java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T16:08:58,358 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1, error=java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:08:58,359 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T16:08:58,359 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 from cache 2024-11-13T16:08:58,359 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address f8f85b0bb1d8:43467 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:09:06,530 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T16:09:08,366 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T16:09:08,366 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1] 2024-11-13T16:09:08,367 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server f8f85b0bb1d8:43467 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:09:08,368 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to f8f85b0bb1d8,43467,1731514088560 for '' in 1588230740 of hbase:meta failed, tries = 12, maxAttempts = 46, timeout = 60000 ms, time elapsed = 48475 ms java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T16:09:08,368 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1, error=java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:09:08,368 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T16:09:08,368 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 from cache 2024-11-13T16:09:08,369 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address f8f85b0bb1d8:43467 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:09:11,538 DEBUG [master/f8f85b0bb1d8:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 93f8ed38b68bd013eb0cf0e7615013ae changed from -1.0 to 0.0, refreshing cache 2024-11-13T16:09:11,539 DEBUG [master/f8f85b0bb1d8:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 34f3d3b6a6dae1f4077567bed0bd8e75 changed from -1.0 to 0.0, refreshing cache 2024-11-13T16:09:11,540 DEBUG [master/f8f85b0bb1d8:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1a610c2403a0b491d368473a6e1fb049 changed from -1.0 to 0.0, refreshing cache 2024-11-13T16:09:11,541 DEBUG [master/f8f85b0bb1d8:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 78285ca993cc7fbfd78041c30558f756 changed from -1.0 to 0.0, refreshing cache 2024-11-13T16:09:11,541 DEBUG [master/f8f85b0bb1d8:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 3647edbd8aef23dda3887b61277da1d3 changed from -1.0 to 0.0, refreshing cache 2024-11-13T16:09:11,541 DEBUG [master/f8f85b0bb1d8:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 7c34e9ded13254b91545186c85befbf8 changed from -1.0 to 0.0, refreshing cache 2024-11-13T16:09:11,542 DEBUG [master/f8f85b0bb1d8:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 97998e7179d85b23a3f5928cd6511629 changed from -1.0 to 0.0, refreshing cache 2024-11-13T16:09:11,542 DEBUG [master/f8f85b0bb1d8:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c4a87adf770b37d40ab6691e377ad1b5 changed from -1.0 to 0.0, refreshing cache 2024-11-13T16:09:11,542 DEBUG [master/f8f85b0bb1d8:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 4a146f98682f25e8cddcae5dbf318603 changed from -1.0 to 0.0, refreshing cache 2024-11-13T16:09:11,542 DEBUG [master/f8f85b0bb1d8:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 4aaf5667f132c289386d1b3b0c0ac57f changed from -1.0 to 0.0, refreshing cache 2024-11-13T16:09:11,543 DEBUG [master/f8f85b0bb1d8:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e72e57b96ff9bc0bbd862ce564d79c42 changed from -1.0 to 0.0, refreshing cache 2024-11-13T16:09:11,543 DEBUG [master/f8f85b0bb1d8:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1fcf78578ccab75f1705f3c2d7bcdb14 changed from -1.0 to 0.0, refreshing cache 2024-11-13T16:09:11,543 DEBUG [master/f8f85b0bb1d8:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region fa6636be3559bc84793534420cd4f207 changed from -1.0 to 0.0, refreshing cache 2024-11-13T16:09:11,543 DEBUG [master/f8f85b0bb1d8:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 05d65f4a6bfecb3d530d15832c218252 changed from -1.0 to 0.0, refreshing cache 2024-11-13T16:09:11,544 DEBUG [master/f8f85b0bb1d8:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region fe97fe0e9b37811ba99ca8100b461589 changed from -1.0 to 0.0, refreshing cache 2024-11-13T16:09:11,544 DEBUG [master/f8f85b0bb1d8:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f8f8b8e4ae8470fd8a1d2bf0684947ef changed from -1.0 to 0.0, refreshing cache 2024-11-13T16:09:11,544 DEBUG [master/f8f85b0bb1d8:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region bf1f9f9bc1bc9ffce566d41f8ae3619a changed from -1.0 to 0.0, refreshing cache 2024-11-13T16:09:11,544 DEBUG [master/f8f85b0bb1d8:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 72d273770314688e78624993de4f3398 changed from -1.0 to 0.0, refreshing cache 2024-11-13T16:09:11,978 INFO [regionserver/f8f85b0bb1d8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: ExecutorStatusChore was stopped 2024-11-13T16:09:15,197 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=f8f85b0bb1d8:42949 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 10 more 2024-11-13T16:09:19,897 WARN [Time-limited test {}] procedure.MasterProcedureTestingUtility$3(130): Failed to load meta org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=13, exceptions: 2024-11-13T16:08:19.900Z, java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:08:20.008Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=f8f85b0bb1d8:43467 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:43467 2024-11-13T16:08:20.218Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=f8f85b0bb1d8:43467 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:43467 2024-11-13T16:08:20.527Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=f8f85b0bb1d8:43467 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:43467 2024-11-13T16:08:21.037Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=f8f85b0bb1d8:43467 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:43467 2024-11-13T16:08:22.063Z, java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:08:24.070Z, java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:08:28.099Z, java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:08:38.179Z, java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:08:48.260Z, java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:08:58.359Z, java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:09:08.368Z, java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:493) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:461) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:422) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.fullScanRegions(MetaTableAccessor.java:108) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionStateStore.visitMeta(RegionStateStore.java:108) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.loadMeta(AssignmentManager.java:1907) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.joinCluster(AssignmentManager.java:1769) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:126) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:121) ~[test-classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:143) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:477) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:486) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPBase.testRecoveryAndDoubleExecution(TestSCPBase.java:114) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPWithMeta.testRecoveryAndDoubleExecutionOnRsWithMeta(TestSCPWithMeta.java:36) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:102) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-13T16:09:19,899 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 1 exec state=pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, hasLock=true; ServerCrashProcedure f8f85b0bb1d8,43467,1731514088560, splitWal=true, meta=true 2024-11-13T16:09:19,899 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-13T16:09:19,899 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T16:09:19,899 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T16:09:19,899 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T16:09:19,900 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-13T16:09:19,900 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-13T16:09:19,900 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-13T16:09:19,900 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T16:09:19,901 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-13T16:09:19,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-13T16:09:19,903 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T16:09:19,904 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-13T16:09:19,904 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=false; ServerCrashProcedure f8f85b0bb1d8,43467,1731514088560, splitWal=true, meta=true 2024-11-13T16:09:19,904 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=false; ServerCrashProcedure f8f85b0bb1d8,43467,1731514088560, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-13T16:09:19,904 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-13T16:09:19,905 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=f8f85b0bb1d8,43467,1731514088560, regionLocation=f8f85b0bb1d8,43467,1731514088560, openSeqNum=2 2024-11-13T16:09:19,905 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=f8f85b0bb1d8,43467,1731514088560, table=hbase:meta, region=1588230740 2024-11-13T16:09:19,906 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f8f85b0bb1d8,43467,1731514088560, state=OPEN 2024-11-13T16:09:19,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42949-0x1013621fa650000, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T16:09:19,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41333-0x1013621fa650003, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T16:09:19,916 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38187-0x1013621fa650002, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T16:09:19,916 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:09:19,916 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:09:19,916 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:09:19,917 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731514189917 2024-11-13T16:09:19,917 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-13T16:09:19,917 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure f8f85b0bb1d8,43467,1731514088560, splitWal=true, meta=true, isMeta: true 2024-11-13T16:09:19,917 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T16:09:19,917 DEBUG [Time-limited test {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1] 2024-11-13T16:09:19,918 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server f8f85b0bb1d8:43467 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:09:19,919 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1, error=java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:09:19,919 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T16:09:19,919 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 from cache 2024-11-13T16:09:19,919 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address f8f85b0bb1d8:43467 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:09:19,920 INFO [PEWorker-1 {}] master.SplitWALManager(105): f8f85b0bb1d8,43467,1731514088560 WAL count=1, meta=true 2024-11-13T16:09:19,920 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure f8f85b0bb1d8%2C43467%2C1731514088560.meta.1731514091182.meta}] 2024-11-13T16:09:19,920 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-13T16:09:19,926 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=f8f85b0bb1d8,38187,1731514088725 2024-11-13T16:09:19,929 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure f8f85b0bb1d8%2C43467%2C1731514088560.meta.1731514091182.meta, worker=f8f85b0bb1d8,38187,1731514088725}] 2024-11-13T16:09:20,026 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T16:09:20,026 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1] 2024-11-13T16:09:20,027 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to f8f85b0bb1d8:43467 this server is in the failed servers list 2024-11-13T16:09:20,028 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=f8f85b0bb1d8:43467 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:43467 2024-11-13T16:09:20,028 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:43467 2024-11-13T16:09:20,028 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 from cache 2024-11-13T16:09:20,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=59 2024-11-13T16:09:20,130 INFO [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(299): Splitting hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/WALs/f8f85b0bb1d8,43467,1731514088560-splitting/f8f85b0bb1d8%2C43467%2C1731514088560.meta.1731514091182.meta, size=52.9 K (54214bytes) 2024-11-13T16:09:20,131 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-13T16:09:20,131 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-13T16:09:20,132 INFO [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/WALs/f8f85b0bb1d8,43467,1731514088560-splitting/f8f85b0bb1d8%2C43467%2C1731514088560.meta.1731514091182.meta 2024-11-13T16:09:20,139 INFO [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/WALs/f8f85b0bb1d8,43467,1731514088560-splitting/f8f85b0bb1d8%2C43467%2C1731514088560.meta.1731514091182.meta after 4ms 2024-11-13T16:09:20,148 INFO [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(310): Open hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/WALs/f8f85b0bb1d8,43467,1731514088560-splitting/f8f85b0bb1d8%2C43467%2C1731514088560.meta.1731514091182.meta took 20ms 2024-11-13T16:09:20,160 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 2024-11-13T16:09:20,170 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/WALs/f8f85b0bb1d8,43467,1731514088560-splitting/f8f85b0bb1d8%2C43467%2C1731514088560.meta.1731514091182.meta so closing down 2024-11-13T16:09:20,170 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T16:09:20,182 INFO [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.meta.1731514091182.meta.temp 2024-11-13T16:09:20,195 INFO [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.meta.1731514091182.meta.temp 2024-11-13T16:09:20,201 INFO [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T16:09:20,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741864_1040 (size=53973) 2024-11-13T16:09:20,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741864_1040 (size=53973) 2024-11-13T16:09:20,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741864_1040 (size=53973) 2024-11-13T16:09:20,220 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.meta.1731514091182.meta.temp (wrote 57 edits, skipped 0 edits in 2 ms) 2024-11-13T16:09:20,222 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.meta.1731514091182.meta.temp to hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-13T16:09:20,227 INFO [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(425): Processed 58 edits across 1 Regions in 74 ms; skipped=1; WAL=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/WALs/f8f85b0bb1d8,43467,1731514088560-splitting/f8f85b0bb1d8%2C43467%2C1731514088560.meta.1731514091182.meta, size=52.9 K, length=54214, corrupted=false, cancelled=false 2024-11-13T16:09:20,227 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(428): Completed split of hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/WALs/f8f85b0bb1d8,43467,1731514088560-splitting/f8f85b0bb1d8%2C43467%2C1731514088560.meta.1731514091182.meta, journal: Splitting hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/WALs/f8f85b0bb1d8,43467,1731514088560-splitting/f8f85b0bb1d8%2C43467%2C1731514088560.meta.1731514091182.meta, size=52.9 K (54214bytes) at 1731514160130Finishing writing output for hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/WALs/f8f85b0bb1d8,43467,1731514088560-splitting/f8f85b0bb1d8%2C43467%2C1731514088560.meta.1731514091182.meta so closing down at 1731514160170 (+40 ms)Creating recovered edits writer path=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.meta.1731514091182.meta.temp at 1731514160195 (+25 ms)3 split writer threads finished at 1731514160201 (+6 ms)Closed recovered edits writer path=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.meta.1731514091182.meta.temp (wrote 57 edits, skipped 0 edits in 2 ms) at 1731514160220 (+19 ms)Rename recovered edits hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.meta.1731514091182.meta.temp to hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 at 1731514160223 (+3 ms)Processed 58 edits across 1 Regions in 74 ms; skipped=1; WAL=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/WALs/f8f85b0bb1d8,43467,1731514088560-splitting/f8f85b0bb1d8%2C43467%2C1731514088560.meta.1731514091182.meta, size=52.9 K, length=54214, corrupted=false, cancelled=false at 1731514160227 (+4 ms) 2024-11-13T16:09:20,227 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/WALs/f8f85b0bb1d8,43467,1731514088560-splitting/f8f85b0bb1d8%2C43467%2C1731514088560.meta.1731514091182.meta 2024-11-13T16:09:20,229 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-13T16:09:20,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42949 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-13T16:09:20,236 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T16:09:20,236 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1] 2024-11-13T16:09:20,236 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to f8f85b0bb1d8:43467 this server is in the failed servers list 2024-11-13T16:09:20,237 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=f8f85b0bb1d8:43467 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:43467 2024-11-13T16:09:20,237 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:43467 2024-11-13T16:09:20,237 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 from cache 2024-11-13T16:09:20,239 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/WALs/f8f85b0bb1d8,43467,1731514088560-splitting/f8f85b0bb1d8%2C43467%2C1731514088560.meta.1731514091182.meta to hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/oldWALs 2024-11-13T16:09:20,242 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-13T16:09:20,242 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f8f85b0bb1d8%2C43467%2C1731514088560.meta.1731514091182.meta, worker=f8f85b0bb1d8,38187,1731514088725 in 310 msec 2024-11-13T16:09:20,244 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=f8f85b0bb1d8,38187,1731514088725 2024-11-13T16:09:20,248 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-13T16:09:20,248 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure f8f85b0bb1d8%2C43467%2C1731514088560.meta.1731514091182.meta, worker=f8f85b0bb1d8,38187,1731514088725 in 325 msec 2024-11-13T16:09:20,250 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/WALs/f8f85b0bb1d8,43467,1731514088560-splitting dir is empty, no logs to split. 2024-11-13T16:09:20,251 INFO [PEWorker-1 {}] master.SplitWALManager(105): f8f85b0bb1d8,43467,1731514088560 WAL count=0, meta=true 2024-11-13T16:09:20,251 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if f8f85b0bb1d8,43467,1731514088560 WAL splitting is done? wals=0, meta=true 2024-11-13T16:09:20,251 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-13T16:09:20,251 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN_META, hasLock=true; ServerCrashProcedure f8f85b0bb1d8,43467,1731514088560, splitWal=true, meta=true 2024-11-13T16:09:20,251 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T16:09:20,251 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN_META, hasLock=true; ServerCrashProcedure f8f85b0bb1d8,43467,1731514088560, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T16:09:20,546 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T16:09:20,546 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1] 2024-11-13T16:09:20,547 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to f8f85b0bb1d8:43467 this server is in the failed servers list 2024-11-13T16:09:20,547 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=f8f85b0bb1d8:43467 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:43467 2024-11-13T16:09:20,547 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:43467 2024-11-13T16:09:20,547 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 from cache 2024-11-13T16:09:21,056 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T16:09:21,056 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1] 2024-11-13T16:09:21,056 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to f8f85b0bb1d8:43467 this server is in the failed servers list 2024-11-13T16:09:21,056 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=f8f85b0bb1d8:43467 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:43467 2024-11-13T16:09:21,056 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:43467 2024-11-13T16:09:21,056 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 from cache 2024-11-13T16:09:22,066 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T16:09:22,066 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1] 2024-11-13T16:09:22,067 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server f8f85b0bb1d8:43467 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:09:22,068 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to f8f85b0bb1d8,43467,1731514088560 for '' in 1588230740 of hbase:meta failed, tries = 6, maxAttempts = 46, timeout = 60000 ms, time elapsed = 2150 ms java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T16:09:22,068 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1, error=java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:09:22,068 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T16:09:22,068 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 from cache 2024-11-13T16:09:22,068 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address f8f85b0bb1d8:43467 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:09:24,086 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T16:09:24,086 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1] 2024-11-13T16:09:24,088 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server f8f85b0bb1d8:43467 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:09:24,089 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to f8f85b0bb1d8,43467,1731514088560 for '' in 1588230740 of hbase:meta failed, tries = 7, maxAttempts = 46, timeout = 60000 ms, time elapsed = 4171 ms java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T16:09:24,089 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1, error=java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:09:24,090 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T16:09:24,090 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 from cache 2024-11-13T16:09:24,090 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address f8f85b0bb1d8:43467 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:09:28,106 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T16:09:28,106 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1] 2024-11-13T16:09:28,107 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server f8f85b0bb1d8:43467 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:09:28,108 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to f8f85b0bb1d8,43467,1731514088560 for '' in 1588230740 of hbase:meta failed, tries = 8, maxAttempts = 46, timeout = 60000 ms, time elapsed = 8190 ms java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T16:09:28,108 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1, error=java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:09:28,108 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T16:09:28,108 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 from cache 2024-11-13T16:09:28,108 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address f8f85b0bb1d8:43467 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:09:36,531 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T16:09:38,126 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T16:09:38,126 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1] 2024-11-13T16:09:38,127 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server f8f85b0bb1d8:43467 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:09:38,128 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to f8f85b0bb1d8,43467,1731514088560 for '' in 1588230740 of hbase:meta failed, tries = 9, maxAttempts = 46, timeout = 60000 ms, time elapsed = 18210 ms java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T16:09:38,128 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1, error=java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:09:38,128 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T16:09:38,128 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 from cache 2024-11-13T16:09:38,128 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address f8f85b0bb1d8:43467 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:09:45,467 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb6ab7af76ac8a0ac with lease ID 0xe17620f2bfeeb2b8: from storage DS-b848e0f5-c5b4-4830-9d9c-fc9502100de9 node DatanodeRegistration(127.0.0.1:45743, datanodeUuid=3de4e64d-0c3f-4dd4-b7a5-e67576d85d5e, infoPort=46455, infoSecurePort=0, ipcPort=33353, storageInfo=lv=-57;cid=testClusterID;nsid=1240587590;c=1731514082621), blocks: 20, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-13T16:09:45,467 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb6ab7af76ac8a0ac with lease ID 0xe17620f2bfeeb2b8: from storage DS-fd3ddb02-0401-4398-a2bc-f4c7aca6fd63 node DatanodeRegistration(127.0.0.1:45743, datanodeUuid=3de4e64d-0c3f-4dd4-b7a5-e67576d85d5e, infoPort=46455, infoSecurePort=0, ipcPort=33353, storageInfo=lv=-57;cid=testClusterID;nsid=1240587590;c=1731514082621), blocks: 20, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T16:09:48,176 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T16:09:48,176 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1] 2024-11-13T16:09:48,178 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server f8f85b0bb1d8:43467 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:09:48,179 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to f8f85b0bb1d8,43467,1731514088560 for '' in 1588230740 of hbase:meta failed, tries = 10, maxAttempts = 46, timeout = 60000 ms, time elapsed = 28261 ms java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T16:09:48,179 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1, error=java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:09:48,180 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T16:09:48,180 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 from cache 2024-11-13T16:09:48,180 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address f8f85b0bb1d8:43467 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:09:58,286 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T16:09:58,287 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1] 2024-11-13T16:09:58,289 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server f8f85b0bb1d8:43467 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:09:58,290 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to f8f85b0bb1d8,43467,1731514088560 for '' in 1588230740 of hbase:meta failed, tries = 11, maxAttempts = 46, timeout = 60000 ms, time elapsed = 38372 ms java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T16:09:58,291 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1, error=java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:09:58,292 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T16:09:58,292 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 from cache 2024-11-13T16:09:58,293 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address f8f85b0bb1d8:43467 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:10:06,531 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T16:10:08,366 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T16:10:08,367 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1] 2024-11-13T16:10:08,367 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server f8f85b0bb1d8:43467 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:10:08,368 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to f8f85b0bb1d8,43467,1731514088560 for '' in 1588230740 of hbase:meta failed, tries = 12, maxAttempts = 46, timeout = 60000 ms, time elapsed = 48450 ms java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T16:10:08,368 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1, error=java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:10:08,368 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T16:10:08,368 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 from cache 2024-11-13T16:10:08,368 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address f8f85b0bb1d8:43467 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:10:10,685 WARN [regionserver/f8f85b0bb1d8:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 25, running: 0 2024-11-13T16:10:10,690 WARN [regionserver/f8f85b0bb1d8:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 17, running: 1 2024-11-13T16:10:10,944 INFO [regionserver/f8f85b0bb1d8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactedHFilesCleaner was stopped 2024-11-13T16:10:19,926 WARN [Time-limited test {}] procedure.MasterProcedureTestingUtility$3(130): Failed to load meta org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=13, exceptions: 2024-11-13T16:09:19.919Z, java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:09:20.028Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=f8f85b0bb1d8:43467 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:43467 2024-11-13T16:09:20.237Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=f8f85b0bb1d8:43467 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:43467 2024-11-13T16:09:20.547Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=f8f85b0bb1d8:43467 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:43467 2024-11-13T16:09:21.057Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=f8f85b0bb1d8:43467 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:43467 2024-11-13T16:09:22.068Z, java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:09:24.090Z, java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:09:28.108Z, java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:09:38.128Z, java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:09:48.180Z, java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:09:58.292Z, java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:10:08.368Z, java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:493) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:461) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:422) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.fullScanRegions(MetaTableAccessor.java:108) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionStateStore.visitMeta(RegionStateStore.java:108) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.loadMeta(AssignmentManager.java:1907) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.joinCluster(AssignmentManager.java:1769) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:126) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:121) ~[test-classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:143) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:477) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:486) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPBase.testRecoveryAndDoubleExecution(TestSCPBase.java:114) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPWithMeta.testRecoveryAndDoubleExecutionOnRsWithMeta(TestSCPWithMeta.java:36) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:102) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-13T16:10:19,928 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 2 exec state=pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN_META, hasLock=true; ServerCrashProcedure f8f85b0bb1d8,43467,1731514088560, splitWal=true, meta=true 2024-11-13T16:10:19,928 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-13T16:10:19,928 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T16:10:19,928 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T16:10:19,928 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T16:10:19,928 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-13T16:10:19,928 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-13T16:10:19,928 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-13T16:10:19,929 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T16:10:19,930 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-13T16:10:19,933 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-13T16:10:19,933 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T16:10:19,933 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T16:10:19,933 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T16:10:19,934 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-13T16:10:19,935 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, hasLock=false; ServerCrashProcedure f8f85b0bb1d8,43467,1731514088560, splitWal=true, meta=true 2024-11-13T16:10:19,935 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure f8f85b0bb1d8%2C43467%2C1731514088560.meta.1731514091182.meta, worker=f8f85b0bb1d8,38187,1731514088725 2024-11-13T16:10:19,935 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f8f85b0bb1d8%2C43467%2C1731514088560.meta.1731514091182.meta, worker=f8f85b0bb1d8,38187,1731514088725 2024-11-13T16:10:19,935 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, hasLock=false; ServerCrashProcedure f8f85b0bb1d8,43467,1731514088560, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-13T16:10:19,935 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure f8f85b0bb1d8%2C43467%2C1731514088560.meta.1731514091182.meta, worker=f8f85b0bb1d8,38187,1731514088725 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T16:10:19,935 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f8f85b0bb1d8%2C43467%2C1731514088560.meta.1731514091182.meta, worker=f8f85b0bb1d8,38187,1731514088725 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T16:10:19,935 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-13T16:10:19,939 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=f8f85b0bb1d8,43467,1731514088560, regionLocation=f8f85b0bb1d8,43467,1731514088560, openSeqNum=2 2024-11-13T16:10:19,939 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=f8f85b0bb1d8,43467,1731514088560, table=hbase:meta, region=1588230740 2024-11-13T16:10:19,939 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f8f85b0bb1d8,43467,1731514088560, state=OPEN 2024-11-13T16:10:19,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41333-0x1013621fa650003, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T16:10:19,997 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38187-0x1013621fa650002, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T16:10:19,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42949-0x1013621fa650000, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T16:10:19,997 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:10:19,997 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:10:19,997 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:10:19,998 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731514249998 2024-11-13T16:10:19,998 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-13T16:10:19,998 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T16:10:19,998 DEBUG [Time-limited test {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1] 2024-11-13T16:10:19,999 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server f8f85b0bb1d8:43467 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:10:19,999 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1, error=java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:10:19,999 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T16:10:20,000 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 from cache 2024-11-13T16:10:20,000 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/WALs/f8f85b0bb1d8,43467,1731514088560-splitting dir is empty, no logs to split. 2024-11-13T16:10:20,000 INFO [PEWorker-1 {}] master.SplitWALManager(105): f8f85b0bb1d8,43467,1731514088560 WAL count=0, meta=true 2024-11-13T16:10:20,000 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address f8f85b0bb1d8:43467 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:10:20,000 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if f8f85b0bb1d8,43467,1731514088560 WAL splitting is done? wals=0, meta=true 2024-11-13T16:10:20,000 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-13T16:10:20,002 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-13T16:10:20,002 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-13T16:10:20,002 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_GET_REGIONS, hasLock=true; ServerCrashProcedure f8f85b0bb1d8,43467,1731514088560, splitWal=true, meta=true 2024-11-13T16:10:20,002 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T16:10:20,002 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_GET_REGIONS, hasLock=true; ServerCrashProcedure f8f85b0bb1d8,43467,1731514088560, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T16:10:20,106 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T16:10:20,106 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1] 2024-11-13T16:10:20,106 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to f8f85b0bb1d8:43467 this server is in the failed servers list 2024-11-13T16:10:20,106 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=f8f85b0bb1d8:43467 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:43467 2024-11-13T16:10:20,106 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:43467 2024-11-13T16:10:20,106 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 from cache 2024-11-13T16:10:20,316 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T16:10:20,316 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1] 2024-11-13T16:10:20,317 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to f8f85b0bb1d8:43467 this server is in the failed servers list 2024-11-13T16:10:20,317 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=f8f85b0bb1d8:43467 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:43467 2024-11-13T16:10:20,317 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:43467 2024-11-13T16:10:20,317 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 from cache 2024-11-13T16:10:20,626 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T16:10:20,626 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1] 2024-11-13T16:10:20,626 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to f8f85b0bb1d8:43467 this server is in the failed servers list 2024-11-13T16:10:20,627 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=f8f85b0bb1d8:43467 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:43467 2024-11-13T16:10:20,627 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:43467 2024-11-13T16:10:20,627 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 from cache 2024-11-13T16:10:21,136 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T16:10:21,136 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1] 2024-11-13T16:10:21,136 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to f8f85b0bb1d8:43467 this server is in the failed servers list 2024-11-13T16:10:21,137 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=f8f85b0bb1d8:43467 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:43467 2024-11-13T16:10:21,137 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:43467 2024-11-13T16:10:21,137 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 from cache 2024-11-13T16:10:22,146 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T16:10:22,147 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1] 2024-11-13T16:10:22,148 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server f8f85b0bb1d8:43467 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:10:22,148 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to f8f85b0bb1d8,43467,1731514088560 for '' in 1588230740 of hbase:meta failed, tries = 6, maxAttempts = 46, timeout = 60000 ms, time elapsed = 2150 ms java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T16:10:22,149 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1, error=java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:10:22,149 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T16:10:22,149 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 from cache 2024-11-13T16:10:22,149 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address f8f85b0bb1d8:43467 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:10:22,199 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=f8f85b0bb1d8:42949 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 10 more 2024-11-13T16:10:24,156 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T16:10:24,156 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1] 2024-11-13T16:10:24,157 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server f8f85b0bb1d8:43467 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:10:24,158 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to f8f85b0bb1d8,43467,1731514088560 for '' in 1588230740 of hbase:meta failed, tries = 7, maxAttempts = 46, timeout = 60000 ms, time elapsed = 4159 ms java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T16:10:24,158 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1, error=java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:10:24,158 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T16:10:24,158 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 from cache 2024-11-13T16:10:24,158 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address f8f85b0bb1d8:43467 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:10:28,186 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T16:10:28,187 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1] 2024-11-13T16:10:28,187 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server f8f85b0bb1d8:43467 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:10:28,188 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to f8f85b0bb1d8,43467,1731514088560 for '' in 1588230740 of hbase:meta failed, tries = 8, maxAttempts = 46, timeout = 60000 ms, time elapsed = 8189 ms java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T16:10:28,188 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1, error=java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:10:28,188 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T16:10:28,188 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 from cache 2024-11-13T16:10:28,189 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address f8f85b0bb1d8:43467 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:10:36,531 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T16:10:38,247 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T16:10:38,247 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1] 2024-11-13T16:10:38,249 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server f8f85b0bb1d8:43467 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:10:38,250 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to f8f85b0bb1d8,43467,1731514088560 for '' in 1588230740 of hbase:meta failed, tries = 9, maxAttempts = 46, timeout = 60000 ms, time elapsed = 18251 ms java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T16:10:38,251 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1, error=java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:10:38,251 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T16:10:38,251 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 from cache 2024-11-13T16:10:38,251 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address f8f85b0bb1d8:43467 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:10:43,461 WARN [RS:2;f8f85b0bb1d8:41333 {}] monitoring.TaskMonitor(166): Status Processing ServerCrashProcedure of f8f85b0bb1d8,43467,1731514088560: status=Processing ServerCrashProcedure of f8f85b0bb1d8,43467,1731514088560 current State SERVER_CRASH_SPLIT_META_LOGS, state=RUNNING, startTime=1731514099848, completionTime=-1 appears to have been leaked 2024-11-13T16:10:43,461 WARN [RS:2;f8f85b0bb1d8:41333 {}] monitoring.TaskMonitor(166): Status Processing ServerCrashProcedure of f8f85b0bb1d8,43467,1731514088560: status=Processing ServerCrashProcedure of f8f85b0bb1d8,43467,1731514088560 current State SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, state=RUNNING, startTime=1731514159904, completionTime=-1 appears to have been leaked 2024-11-13T16:10:48,287 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T16:10:48,287 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1] 2024-11-13T16:10:48,288 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server f8f85b0bb1d8:43467 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:10:48,288 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to f8f85b0bb1d8,43467,1731514088560 for '' in 1588230740 of hbase:meta failed, tries = 10, maxAttempts = 46, timeout = 60000 ms, time elapsed = 28290 ms java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T16:10:48,289 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1, error=java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:10:48,289 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T16:10:48,289 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 from cache 2024-11-13T16:10:48,289 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address f8f85b0bb1d8:43467 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:10:58,366 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T16:10:58,366 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1] 2024-11-13T16:10:58,367 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server f8f85b0bb1d8:43467 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:10:58,368 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to f8f85b0bb1d8,43467,1731514088560 for '' in 1588230740 of hbase:meta failed, tries = 11, maxAttempts = 46, timeout = 60000 ms, time elapsed = 38369 ms java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T16:10:58,368 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1, error=java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:10:58,368 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T16:10:58,369 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 from cache 2024-11-13T16:10:58,369 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address f8f85b0bb1d8:43467 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:11:06,532 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T16:11:08,446 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T16:11:08,447 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1] 2024-11-13T16:11:08,448 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server f8f85b0bb1d8:43467 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:11:08,448 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to f8f85b0bb1d8,43467,1731514088560 for '' in 1588230740 of hbase:meta failed, tries = 12, maxAttempts = 46, timeout = 60000 ms, time elapsed = 48450 ms java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T16:11:08,448 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1, error=java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:11:08,449 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T16:11:08,449 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 from cache 2024-11-13T16:11:08,449 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address f8f85b0bb1d8:43467 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:11:20,007 WARN [Time-limited test {}] procedure.MasterProcedureTestingUtility$3(130): Failed to load meta org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=13, exceptions: 2024-11-13T16:10:20Z, java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:10:20.106Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=f8f85b0bb1d8:43467 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:43467 2024-11-13T16:10:20.317Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=f8f85b0bb1d8:43467 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:43467 2024-11-13T16:10:20.627Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=f8f85b0bb1d8:43467 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:43467 2024-11-13T16:10:21.137Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=f8f85b0bb1d8:43467 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:43467 2024-11-13T16:10:22.149Z, java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:10:24.158Z, java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:10:28.188Z, java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:10:38.251Z, java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:10:48.289Z, java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:10:58.369Z, java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:11:08.449Z, java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:493) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:461) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:422) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.fullScanRegions(MetaTableAccessor.java:108) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionStateStore.visitMeta(RegionStateStore.java:108) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.loadMeta(AssignmentManager.java:1907) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.joinCluster(AssignmentManager.java:1769) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:126) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:121) ~[test-classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:143) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:477) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:486) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPBase.testRecoveryAndDoubleExecution(TestSCPBase.java:114) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPWithMeta.testRecoveryAndDoubleExecutionOnRsWithMeta(TestSCPWithMeta.java:36) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:102) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-13T16:11:20,009 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 3 exec state=pid=57, state=WAITING:SERVER_CRASH_GET_REGIONS, hasLock=true; ServerCrashProcedure f8f85b0bb1d8,43467,1731514088560, splitWal=true, meta=true 2024-11-13T16:11:20,009 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-13T16:11:20,009 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T16:11:20,009 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T16:11:20,009 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T16:11:20,009 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-13T16:11:20,009 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-13T16:11:20,009 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-13T16:11:20,010 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T16:11:20,012 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 2 msec 2024-11-13T16:11:20,015 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-13T16:11:20,015 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T16:11:20,015 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T16:11:20,015 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T16:11:20,017 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-13T16:11:20,017 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN_META, hasLock=false; ServerCrashProcedure f8f85b0bb1d8,43467,1731514088560, splitWal=true, meta=true 2024-11-13T16:11:20,017 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure f8f85b0bb1d8%2C43467%2C1731514088560.meta.1731514091182.meta, worker=f8f85b0bb1d8,38187,1731514088725 2024-11-13T16:11:20,017 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f8f85b0bb1d8%2C43467%2C1731514088560.meta.1731514091182.meta, worker=f8f85b0bb1d8,38187,1731514088725 2024-11-13T16:11:20,017 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN_META, hasLock=false; ServerCrashProcedure f8f85b0bb1d8,43467,1731514088560, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-13T16:11:20,018 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure f8f85b0bb1d8%2C43467%2C1731514088560.meta.1731514091182.meta, worker=f8f85b0bb1d8,38187,1731514088725 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T16:11:20,018 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f8f85b0bb1d8%2C43467%2C1731514088560.meta.1731514091182.meta, worker=f8f85b0bb1d8,38187,1731514088725 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T16:11:20,018 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-13T16:11:20,019 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=f8f85b0bb1d8,43467,1731514088560, regionLocation=f8f85b0bb1d8,43467,1731514088560, openSeqNum=2 2024-11-13T16:11:20,019 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=f8f85b0bb1d8,43467,1731514088560, table=hbase:meta, region=1588230740 2024-11-13T16:11:20,019 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f8f85b0bb1d8,43467,1731514088560, state=OPEN 2024-11-13T16:11:20,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42949-0x1013621fa650000, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T16:11:20,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41333-0x1013621fa650003, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T16:11:20,059 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38187-0x1013621fa650002, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T16:11:20,060 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:11:20,060 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:11:20,060 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:11:20,067 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731514310066 2024-11-13T16:11:20,067 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-13T16:11:20,067 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T16:11:20,067 DEBUG [Time-limited test {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1] 2024-11-13T16:11:20,068 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server f8f85b0bb1d8:43467 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:11:20,069 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1, error=java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:11:20,069 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T16:11:20,069 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 from cache 2024-11-13T16:11:20,069 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address f8f85b0bb1d8:43467 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:11:20,071 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-13T16:11:20,071 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-13T16:11:20,075 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T16:11:20,076 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T16:11:20,176 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T16:11:20,176 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1] 2024-11-13T16:11:20,177 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to f8f85b0bb1d8:43467 this server is in the failed servers list 2024-11-13T16:11:20,177 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=f8f85b0bb1d8:43467 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:43467 2024-11-13T16:11:20,177 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:43467 2024-11-13T16:11:20,177 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 from cache 2024-11-13T16:11:20,227 DEBUG [f8f85b0bb1d8:42949 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-13T16:11:20,227 DEBUG [f8f85b0bb1d8:42949 {}] balancer.BalancerClusterState(204): Hosts are {f8f85b0bb1d8=0} racks are {/default-rack=0} 2024-11-13T16:11:20,227 DEBUG [f8f85b0bb1d8:42949 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T16:11:20,227 DEBUG [f8f85b0bb1d8:42949 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T16:11:20,227 DEBUG [f8f85b0bb1d8:42949 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T16:11:20,227 DEBUG [f8f85b0bb1d8:42949 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T16:11:20,227 INFO [f8f85b0bb1d8:42949 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T16:11:20,227 INFO [f8f85b0bb1d8:42949 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T16:11:20,227 DEBUG [f8f85b0bb1d8:42949 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T16:11:20,228 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:11:20,229 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f8f85b0bb1d8,41333,1731514088798, state=OPENING 2024-11-13T16:11:20,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41333-0x1013621fa650003, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T16:11:20,259 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38187-0x1013621fa650002, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T16:11:20,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42949-0x1013621fa650000, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T16:11:20,260 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T16:11:20,260 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:11:20,260 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:11:20,260 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:11:20,260 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=f8f85b0bb1d8,41333,1731514088798}] 2024-11-13T16:11:20,386 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T16:11:20,386 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=-1] 2024-11-13T16:11:20,387 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T16:11:20,389 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35953, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T16:11:20,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41333 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Scan size: 121 connection: 172.17.0.3:35953 deadline: 1731514340069, exception=org.apache.hadoop.hbase.NotServingRegionException: hbase:meta,,1 is not online on f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:11:20,394 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=-1, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: hbase:meta,,1 is not online on f8f85b0bb1d8,41333,1731514088798 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T16:11:20,394 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=-1 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: hbase:meta,,1 is not online on f8f85b0bb1d8,41333,1731514088798 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T16:11:20,394 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=-1 from cache 2024-11-13T16:11:20,415 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60719, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T16:11:20,423 INFO [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-13T16:11:20,423 INFO [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T16:11:20,423 INFO [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=61}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-13T16:11:20,435 INFO [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f8f85b0bb1d8%2C41333%2C1731514088798.meta, suffix=.meta, logDir=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/WALs/f8f85b0bb1d8,41333,1731514088798, archiveDir=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/oldWALs, maxLogs=32 2024-11-13T16:11:20,456 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=61}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/WALs/f8f85b0bb1d8,41333,1731514088798/f8f85b0bb1d8%2C41333%2C1731514088798.meta.1731514280437.meta, exclude list is [], retry=0 2024-11-13T16:11:20,466 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32863,DS-af4262a8-b437-4e57-821f-0315c01636b9,DISK] 2024-11-13T16:11:20,467 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33087,DS-83f68ee5-197b-4527-968f-c0dc23954597,DISK] 2024-11-13T16:11:20,467 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45743,DS-b848e0f5-c5b4-4830-9d9c-fc9502100de9,DISK] 2024-11-13T16:11:20,480 INFO [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/WALs/f8f85b0bb1d8,41333,1731514088798/f8f85b0bb1d8%2C41333%2C1731514088798.meta.1731514280437.meta 2024-11-13T16:11:20,483 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33713:33713),(127.0.0.1/127.0.0.1:38919:38919),(127.0.0.1/127.0.0.1:46455:46455)] 2024-11-13T16:11:20,484 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-13T16:11:20,484 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=61}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-13T16:11:20,484 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-13T16:11:20,484 INFO [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-13T16:11:20,485 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-13T16:11:20,485 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:11:20,485 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-13T16:11:20,485 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-13T16:11:20,499 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T16:11:20,507 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T16:11:20,507 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:11:20,515 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T16:11:20,515 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T16:11:20,524 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T16:11:20,525 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:11:20,527 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T16:11:20,527 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T16:11:20,532 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T16:11:20,532 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:11:20,538 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T16:11:20,538 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T16:11:20,547 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T16:11:20,547 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:11:20,548 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T16:11:20,548 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T16:11:20,549 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/hbase/meta/1588230740 2024-11-13T16:11:20,553 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/hbase/meta/1588230740 2024-11-13T16:11:20,563 INFO [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-13T16:11:20,589 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5793): Applied 264, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=60, path=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-13T16:11:20,594 INFO [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=49.58 KB heapSize=69.98 KB 2024-11-13T16:11:20,706 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T16:11:20,707 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=-1] 2024-11-13T16:11:20,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41333 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Scan size: 121 connection: 172.17.0.3:35953 deadline: 1731514340068, exception=org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:11:20,709 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=-1, error=org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on f8f85b0bb1d8,41333,1731514088798 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T16:11:20,710 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=-1 is org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on f8f85b0bb1d8,41333,1731514088798 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T16:11:20,710 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=-1 from cache 2024-11-13T16:11:20,728 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/hbase/meta/1588230740/.tmp/info/087b194b55e94b9882f7894cb43e70cc is 268, key is testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731514091776.8ba796d80fc3b65f0948c95ccbdad39d./info:regioninfo/1731514093683/Put/seqid=0 2024-11-13T16:11:20,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741866_1042 (size=58053) 2024-11-13T16:11:20,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741866_1042 (size=58053) 2024-11-13T16:11:20,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741866_1042 (size=58053) 2024-11-13T16:11:20,789 INFO [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.31 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/hbase/meta/1588230740/.tmp/info/087b194b55e94b9882f7894cb43e70cc 2024-11-13T16:11:20,919 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/hbase/meta/1588230740/.tmp/ns/a444dc7489f94a26a35001951d90ac59 is 43, key is default/ns:d/1731514091403/Put/seqid=0 2024-11-13T16:11:21,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741867_1043 (size=5153) 2024-11-13T16:11:21,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741867_1043 (size=5153) 2024-11-13T16:11:21,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741867_1043 (size=5153) 2024-11-13T16:11:21,023 INFO [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/hbase/meta/1588230740/.tmp/ns/a444dc7489f94a26a35001951d90ac59 2024-11-13T16:11:21,111 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/hbase/meta/1588230740/.tmp/table/173617d476ce4ecba2595b8551353aa9 is 105, key is testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/table:state/1731514094309/Put/seqid=0 2024-11-13T16:11:21,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741868_1044 (size=5620) 2024-11-13T16:11:21,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741868_1044 (size=5620) 2024-11-13T16:11:21,142 INFO [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=202 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/hbase/meta/1588230740/.tmp/table/173617d476ce4ecba2595b8551353aa9 2024-11-13T16:11:21,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741868_1044 (size=5620) 2024-11-13T16:11:21,178 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/hbase/meta/1588230740/.tmp/info/087b194b55e94b9882f7894cb43e70cc as hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/hbase/meta/1588230740/info/087b194b55e94b9882f7894cb43e70cc 2024-11-13T16:11:21,202 INFO [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/hbase/meta/1588230740/info/087b194b55e94b9882f7894cb43e70cc, entries=260, sequenceid=60, filesize=56.7 K 2024-11-13T16:11:21,208 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/hbase/meta/1588230740/.tmp/ns/a444dc7489f94a26a35001951d90ac59 as hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/hbase/meta/1588230740/ns/a444dc7489f94a26a35001951d90ac59 2024-11-13T16:11:21,218 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T16:11:21,219 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=-1] 2024-11-13T16:11:21,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41333 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Scan size: 121 connection: 172.17.0.3:35953 deadline: 1731514340069, exception=org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:11:21,222 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=-1, error=org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on f8f85b0bb1d8,41333,1731514088798 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T16:11:21,222 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=-1 is org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on f8f85b0bb1d8,41333,1731514088798 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T16:11:21,222 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=-1 from cache 2024-11-13T16:11:21,225 INFO [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/hbase/meta/1588230740/ns/a444dc7489f94a26a35001951d90ac59, entries=2, sequenceid=60, filesize=5.0 K 2024-11-13T16:11:21,227 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/hbase/meta/1588230740/.tmp/table/173617d476ce4ecba2595b8551353aa9 as hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/hbase/meta/1588230740/table/173617d476ce4ecba2595b8551353aa9 2024-11-13T16:11:21,242 INFO [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/hbase/meta/1588230740/table/173617d476ce4ecba2595b8551353aa9, entries=2, sequenceid=60, filesize=5.5 K 2024-11-13T16:11:21,254 INFO [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~49.58 KB/50772, heapSize ~69.69 KB/71360, currentSize=0 B/0 for 1588230740 in 652ms, sequenceid=60, compaction requested=false; wal=null 2024-11-13T16:11:21,259 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-13T16:11:21,275 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T16:11:21,275 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T16:11:21,276 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T16:11:21,278 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T16:11:21,283 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/hbase/meta/1588230740/recovered.edits/60.seqid, newMaxSeqId=60, maxSeqId=1 2024-11-13T16:11:21,284 INFO [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=61; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70800457, jitterRate=0.05500902235507965}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T16:11:21,284 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-13T16:11:21,285 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731514280485Writing region info on filesystem at 1731514280485Initializing all the Stores at 1731514280490 (+5 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731514280490Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731514280494 (+4 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514280495 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731514280495Obtaining lock to block concurrent updates at 1731514280594 (+99 ms)Preparing flush snapshotting stores in 1588230740 at 1731514280594Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=50772, getHeapSize=71600, getOffHeapSize=0, getCellsCount=264 at 1731514280598 (+4 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731514280598Flushing 1588230740/info: creating writer at 1731514280603 (+5 ms)Flushing 1588230740/info: appending metadata at 1731514280715 (+112 ms)Flushing 1588230740/info: closing flushed file at 1731514280720 (+5 ms)Flushing 1588230740/ns: creating writer at 1731514280865 (+145 ms)Flushing 1588230740/ns: appending metadata at 1731514280918 (+53 ms)Flushing 1588230740/ns: closing flushed file at 1731514280918Flushing 1588230740/table: creating writer at 1731514281081 (+163 ms)Flushing 1588230740/table: appending metadata at 1731514281106 (+25 ms)Flushing 1588230740/table: closing flushed file at 1731514281106Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3cef547b: reopening flushed file at 1731514281176 (+70 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@574a7846: reopening flushed file at 1731514281205 (+29 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b50b366: reopening flushed file at 1731514281225 (+20 ms)Finished flush of dataSize ~49.58 KB/50772, heapSize ~69.69 KB/71360, currentSize=0 B/0 for 1588230740 in 652ms, sequenceid=60, compaction requested=false; wal=null at 1731514281254 (+29 ms)Cleaning up temporary data from old regions at 1731514281275 (+21 ms)Running coprocessor post-open hooks at 1731514281284 (+9 ms)Region opened successfully at 1731514281284 2024-11-13T16:11:21,289 INFO [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=61, masterSystemTime=1731514280413 2024-11-13T16:11:21,296 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=61, regionLocation=f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:11:21,298 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f8f85b0bb1d8,41333,1731514088798, state=OPEN 2024-11-13T16:11:21,298 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-13T16:11:21,298 INFO [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-13T16:11:21,310 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41333-0x1013621fa650003, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T16:11:21,310 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42949-0x1013621fa650000, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T16:11:21,310 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38187-0x1013621fa650002, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T16:11:21,310 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:11:21,310 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=61, ppid=60, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:11:21,310 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:11:21,310 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:11:21,322 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-13T16:11:21,322 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f8f85b0bb1d8,41333,1731514088798 in 1.0500 sec 2024-11-13T16:11:21,343 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=57 2024-11-13T16:11:21,343 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.2520 sec 2024-11-13T16:11:22,236 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T16:11:22,236 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=-1] 2024-11-13T16:11:22,285 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4aaf5667f132c289386d1b3b0c0ac57f, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=2 2024-11-13T16:11:22,286 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ba796d80fc3b65f0948c95ccbdad39d, regionState=OPEN, lastHost=f8f85b0bb1d8,43467,1731514088560, regionLocation=f8f85b0bb1d8,43467,1731514088560, openSeqNum=2 2024-11-13T16:11:22,286 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93f8ed38b68bd013eb0cf0e7615013ae, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=2 2024-11-13T16:11:22,286 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e72e57b96ff9bc0bbd862ce564d79c42, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=2 2024-11-13T16:11:22,286 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34f3d3b6a6dae1f4077567bed0bd8e75, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=2 2024-11-13T16:11:22,286 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3945ce860c8642051e17517944a83136, regionState=OPEN, lastHost=f8f85b0bb1d8,43467,1731514088560, regionLocation=f8f85b0bb1d8,43467,1731514088560, openSeqNum=2 2024-11-13T16:11:22,287 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1fcf78578ccab75f1705f3c2d7bcdb14, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=2 2024-11-13T16:11:22,287 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a610c2403a0b491d368473a6e1fb049, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=2 2024-11-13T16:11:22,287 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8725a794e6d251e1bf6e3502baa7700, regionState=OPEN, lastHost=f8f85b0bb1d8,43467,1731514088560, regionLocation=f8f85b0bb1d8,43467,1731514088560, openSeqNum=2 2024-11-13T16:11:22,287 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa6636be3559bc84793534420cd4f207, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=2 2024-11-13T16:11:22,287 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=05d65f4a6bfecb3d530d15832c218252, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=2 2024-11-13T16:11:22,287 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78285ca993cc7fbfd78041c30558f756, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=2 2024-11-13T16:11:22,288 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa124b802c708ca1f715cb1f8c9c4eaa, regionState=OPEN, lastHost=f8f85b0bb1d8,43467,1731514088560, regionLocation=f8f85b0bb1d8,43467,1731514088560, openSeqNum=2 2024-11-13T16:11:22,288 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3647edbd8aef23dda3887b61277da1d3, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=2 2024-11-13T16:11:22,288 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=29df624bbcb878edce7fcdf0abceea88, regionState=OPEN, lastHost=f8f85b0bb1d8,43467,1731514088560, regionLocation=f8f85b0bb1d8,43467,1731514088560, openSeqNum=2 2024-11-13T16:11:22,288 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fe97fe0e9b37811ba99ca8100b461589, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=2 2024-11-13T16:11:22,289 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7c34e9ded13254b91545186c85befbf8, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=2 2024-11-13T16:11:22,290 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8f8b8e4ae8470fd8a1d2bf0684947ef, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=2 2024-11-13T16:11:22,290 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fec5f21b48c36bf01685ec2ee58f0490, regionState=OPEN, lastHost=f8f85b0bb1d8,43467,1731514088560, regionLocation=f8f85b0bb1d8,43467,1731514088560, openSeqNum=2 2024-11-13T16:11:22,290 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97998e7179d85b23a3f5928cd6511629, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=2 2024-11-13T16:11:22,290 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=519ece296a21b5ddccc38a7c928780bb, regionState=OPEN, lastHost=f8f85b0bb1d8,43467,1731514088560, regionLocation=f8f85b0bb1d8,43467,1731514088560, openSeqNum=2 2024-11-13T16:11:22,290 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4a87adf770b37d40ab6691e377ad1b5, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=2 2024-11-13T16:11:22,291 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf1f9f9bc1bc9ffce566d41f8ae3619a, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=2 2024-11-13T16:11:22,291 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a146f98682f25e8cddcae5dbf318603, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=2 2024-11-13T16:11:22,291 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6be98d350311c17b11fdb9e9a1238fca, regionState=OPEN, lastHost=f8f85b0bb1d8,43467,1731514088560, regionLocation=f8f85b0bb1d8,43467,1731514088560, openSeqNum=2 2024-11-13T16:11:22,291 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=72d273770314688e78624993de4f3398, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=2 2024-11-13T16:11:22,291 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-13T16:11:22,291 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731514342291 2024-11-13T16:11:22,291 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731514402291 2024-11-13T16:11:22,292 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 2.2240 sec 2024-11-13T16:11:22,292 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): f8f85b0bb1d8,43467,1731514088560 had 9 regions 2024-11-13T16:11:22,293 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-13T16:11:22,293 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure f8f85b0bb1d8,43467,1731514088560, splitWal=true, meta=true 2024-11-13T16:11:22,293 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T16:11:22,293 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure f8f85b0bb1d8,43467,1731514088560, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T16:11:22,542 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 4 exec state=pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure f8f85b0bb1d8,43467,1731514088560, splitWal=true, meta=true 2024-11-13T16:11:22,542 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-13T16:11:22,542 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T16:11:22,542 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T16:11:22,543 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T16:11:22,543 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-13T16:11:22,543 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-13T16:11:22,543 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-13T16:11:22,543 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T16:11:22,544 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-13T16:11:22,547 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-13T16:11:22,547 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T16:11:22,547 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T16:11:22,547 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T16:11:22,547 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T16:11:22,548 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T16:11:22,549 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-13T16:11:22,550 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_GET_REGIONS, hasLock=false; ServerCrashProcedure f8f85b0bb1d8,43467,1731514088560, splitWal=true, meta=true 2024-11-13T16:11:22,550 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure f8f85b0bb1d8%2C43467%2C1731514088560.meta.1731514091182.meta, worker=f8f85b0bb1d8,38187,1731514088725 2024-11-13T16:11:22,550 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f8f85b0bb1d8%2C43467%2C1731514088560.meta.1731514091182.meta, worker=f8f85b0bb1d8,38187,1731514088725 2024-11-13T16:11:22,550 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T16:11:22,550 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:11:22,550 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_GET_REGIONS, hasLock=false; ServerCrashProcedure f8f85b0bb1d8,43467,1731514088560, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-13T16:11:22,551 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure f8f85b0bb1d8%2C43467%2C1731514088560.meta.1731514091182.meta, worker=f8f85b0bb1d8,38187,1731514088725 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T16:11:22,551 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f8f85b0bb1d8%2C43467%2C1731514088560.meta.1731514091182.meta, worker=f8f85b0bb1d8,38187,1731514088725 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T16:11:22,551 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T16:11:22,551 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f8f85b0bb1d8,41333,1731514088798 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T16:11:22,551 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-13T16:11:22,552 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=61 2024-11-13T16:11:22,552 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=f8f85b0bb1d8,41333,1731514088798, table=hbase:meta, region=1588230740 2024-11-13T16:11:22,553 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f8f85b0bb1d8,41333,1731514088798, state=OPEN 2024-11-13T16:11:22,577 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42949 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T16:11:22,600 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42949 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T16:11:22,763 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42949-0x1013621fa650000, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T16:11:22,763 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38187-0x1013621fa650002, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T16:11:22,763 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41333-0x1013621fa650003, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T16:11:22,764 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:11:22,764 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:11:22,764 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:11:22,765 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731514312765 2024-11-13T16:11:22,765 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-13T16:11:22,772 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4aaf5667f132c289386d1b3b0c0ac57f, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=2 2024-11-13T16:11:22,772 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ba796d80fc3b65f0948c95ccbdad39d, regionState=OPEN, lastHost=f8f85b0bb1d8,43467,1731514088560, regionLocation=f8f85b0bb1d8,43467,1731514088560, openSeqNum=2 2024-11-13T16:11:22,772 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93f8ed38b68bd013eb0cf0e7615013ae, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=2 2024-11-13T16:11:22,773 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e72e57b96ff9bc0bbd862ce564d79c42, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=2 2024-11-13T16:11:22,773 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34f3d3b6a6dae1f4077567bed0bd8e75, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=2 2024-11-13T16:11:22,773 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3945ce860c8642051e17517944a83136, regionState=OPEN, lastHost=f8f85b0bb1d8,43467,1731514088560, regionLocation=f8f85b0bb1d8,43467,1731514088560, openSeqNum=2 2024-11-13T16:11:22,773 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1fcf78578ccab75f1705f3c2d7bcdb14, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=2 2024-11-13T16:11:22,773 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a610c2403a0b491d368473a6e1fb049, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=2 2024-11-13T16:11:22,773 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8725a794e6d251e1bf6e3502baa7700, regionState=OPEN, lastHost=f8f85b0bb1d8,43467,1731514088560, regionLocation=f8f85b0bb1d8,43467,1731514088560, openSeqNum=2 2024-11-13T16:11:22,773 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa6636be3559bc84793534420cd4f207, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=2 2024-11-13T16:11:22,774 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=05d65f4a6bfecb3d530d15832c218252, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=2 2024-11-13T16:11:22,774 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78285ca993cc7fbfd78041c30558f756, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=2 2024-11-13T16:11:22,774 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa124b802c708ca1f715cb1f8c9c4eaa, regionState=OPEN, lastHost=f8f85b0bb1d8,43467,1731514088560, regionLocation=f8f85b0bb1d8,43467,1731514088560, openSeqNum=2 2024-11-13T16:11:22,774 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3647edbd8aef23dda3887b61277da1d3, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=2 2024-11-13T16:11:22,774 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=29df624bbcb878edce7fcdf0abceea88, regionState=OPEN, lastHost=f8f85b0bb1d8,43467,1731514088560, regionLocation=f8f85b0bb1d8,43467,1731514088560, openSeqNum=2 2024-11-13T16:11:22,774 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fe97fe0e9b37811ba99ca8100b461589, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=2 2024-11-13T16:11:22,775 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7c34e9ded13254b91545186c85befbf8, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=2 2024-11-13T16:11:22,775 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8f8b8e4ae8470fd8a1d2bf0684947ef, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=2 2024-11-13T16:11:22,775 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fec5f21b48c36bf01685ec2ee58f0490, regionState=OPEN, lastHost=f8f85b0bb1d8,43467,1731514088560, regionLocation=f8f85b0bb1d8,43467,1731514088560, openSeqNum=2 2024-11-13T16:11:22,775 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97998e7179d85b23a3f5928cd6511629, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=2 2024-11-13T16:11:22,775 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=519ece296a21b5ddccc38a7c928780bb, regionState=OPEN, lastHost=f8f85b0bb1d8,43467,1731514088560, regionLocation=f8f85b0bb1d8,43467,1731514088560, openSeqNum=2 2024-11-13T16:11:22,775 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4a87adf770b37d40ab6691e377ad1b5, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=2 2024-11-13T16:11:22,775 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf1f9f9bc1bc9ffce566d41f8ae3619a, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=2 2024-11-13T16:11:22,775 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a146f98682f25e8cddcae5dbf318603, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=2 2024-11-13T16:11:22,776 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6be98d350311c17b11fdb9e9a1238fca, regionState=OPEN, lastHost=f8f85b0bb1d8,43467,1731514088560, regionLocation=f8f85b0bb1d8,43467,1731514088560, openSeqNum=2 2024-11-13T16:11:22,776 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=72d273770314688e78624993de4f3398, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=2 2024-11-13T16:11:22,776 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-13T16:11:22,776 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731514342776 2024-11-13T16:11:22,776 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731514402776 2024-11-13T16:11:22,776 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 11 msec 2024-11-13T16:11:22,776 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): f8f85b0bb1d8,43467,1731514088560 had 8 regions 2024-11-13T16:11:22,776 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-13T16:11:22,778 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure f8f85b0bb1d8,43467,1731514088560, splitWal=true, meta=true, isMeta: false 2024-11-13T16:11:22,780 INFO [PEWorker-1 {}] master.SplitWALManager(105): f8f85b0bb1d8,43467,1731514088560 WAL count=1, meta=false 2024-11-13T16:11:22,780 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856}] 2024-11-13T16:11:22,780 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-13T16:11:22,780 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_WALS_DIR, hasLock=true; ServerCrashProcedure f8f85b0bb1d8,43467,1731514088560, splitWal=true, meta=true 2024-11-13T16:11:22,780 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T16:11:22,780 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_WALS_DIR, hasLock=true; ServerCrashProcedure f8f85b0bb1d8,43467,1731514088560, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T16:11:23,026 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 5 exec state=pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_WALS_DIR, hasLock=true; ServerCrashProcedure f8f85b0bb1d8,43467,1731514088560, splitWal=true, meta=true 2024-11-13T16:11:23,027 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-13T16:11:23,027 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T16:11:23,027 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T16:11:23,027 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T16:11:23,027 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-13T16:11:23,027 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-13T16:11:23,028 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-13T16:11:23,028 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T16:11:23,029 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-13T16:11:23,031 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-13T16:11:23,031 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T16:11:23,031 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T16:11:23,031 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T16:11:23,032 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T16:11:23,032 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T16:11:23,033 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-13T16:11:23,034 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=false; ServerCrashProcedure f8f85b0bb1d8,43467,1731514088560, splitWal=true, meta=true 2024-11-13T16:11:23,034 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure f8f85b0bb1d8%2C43467%2C1731514088560.meta.1731514091182.meta, worker=f8f85b0bb1d8,38187,1731514088725 2024-11-13T16:11:23,034 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f8f85b0bb1d8%2C43467%2C1731514088560.meta.1731514091182.meta, worker=f8f85b0bb1d8,38187,1731514088725 2024-11-13T16:11:23,034 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T16:11:23,034 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:11:23,034 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=false; ServerCrashProcedure f8f85b0bb1d8,43467,1731514088560, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-13T16:11:23,034 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure f8f85b0bb1d8%2C43467%2C1731514088560.meta.1731514091182.meta, worker=f8f85b0bb1d8,38187,1731514088725 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T16:11:23,034 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f8f85b0bb1d8%2C43467%2C1731514088560.meta.1731514091182.meta, worker=f8f85b0bb1d8,38187,1731514088725 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T16:11:23,035 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T16:11:23,035 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f8f85b0bb1d8,41333,1731514088798 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T16:11:23,035 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-13T16:11:23,036 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=61 2024-11-13T16:11:23,037 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=f8f85b0bb1d8,41333,1731514088798, table=hbase:meta, region=1588230740 2024-11-13T16:11:23,037 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f8f85b0bb1d8,41333,1731514088798, state=OPEN 2024-11-13T16:11:23,049 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42949-0x1013621fa650000, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T16:11:23,049 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38187-0x1013621fa650002, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T16:11:23,049 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41333-0x1013621fa650003, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T16:11:23,049 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:11:23,049 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:11:23,049 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:11:23,052 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731514313052 2024-11-13T16:11:23,052 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-13T16:11:23,059 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4aaf5667f132c289386d1b3b0c0ac57f, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=2 2024-11-13T16:11:23,059 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ba796d80fc3b65f0948c95ccbdad39d, regionState=OPEN, lastHost=f8f85b0bb1d8,43467,1731514088560, regionLocation=f8f85b0bb1d8,43467,1731514088560, openSeqNum=2 2024-11-13T16:11:23,060 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93f8ed38b68bd013eb0cf0e7615013ae, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=2 2024-11-13T16:11:23,060 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e72e57b96ff9bc0bbd862ce564d79c42, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=2 2024-11-13T16:11:23,060 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34f3d3b6a6dae1f4077567bed0bd8e75, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=2 2024-11-13T16:11:23,060 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3945ce860c8642051e17517944a83136, regionState=OPEN, lastHost=f8f85b0bb1d8,43467,1731514088560, regionLocation=f8f85b0bb1d8,43467,1731514088560, openSeqNum=2 2024-11-13T16:11:23,060 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1fcf78578ccab75f1705f3c2d7bcdb14, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=2 2024-11-13T16:11:23,060 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a610c2403a0b491d368473a6e1fb049, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=2 2024-11-13T16:11:23,061 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8725a794e6d251e1bf6e3502baa7700, regionState=OPEN, lastHost=f8f85b0bb1d8,43467,1731514088560, regionLocation=f8f85b0bb1d8,43467,1731514088560, openSeqNum=2 2024-11-13T16:11:23,061 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa6636be3559bc84793534420cd4f207, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=2 2024-11-13T16:11:23,061 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=05d65f4a6bfecb3d530d15832c218252, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=2 2024-11-13T16:11:23,061 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78285ca993cc7fbfd78041c30558f756, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=2 2024-11-13T16:11:23,061 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa124b802c708ca1f715cb1f8c9c4eaa, regionState=OPEN, lastHost=f8f85b0bb1d8,43467,1731514088560, regionLocation=f8f85b0bb1d8,43467,1731514088560, openSeqNum=2 2024-11-13T16:11:23,061 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3647edbd8aef23dda3887b61277da1d3, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=2 2024-11-13T16:11:23,061 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=29df624bbcb878edce7fcdf0abceea88, regionState=OPEN, lastHost=f8f85b0bb1d8,43467,1731514088560, regionLocation=f8f85b0bb1d8,43467,1731514088560, openSeqNum=2 2024-11-13T16:11:23,062 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fe97fe0e9b37811ba99ca8100b461589, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=2 2024-11-13T16:11:23,062 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7c34e9ded13254b91545186c85befbf8, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=2 2024-11-13T16:11:23,062 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8f8b8e4ae8470fd8a1d2bf0684947ef, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=2 2024-11-13T16:11:23,062 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fec5f21b48c36bf01685ec2ee58f0490, regionState=OPEN, lastHost=f8f85b0bb1d8,43467,1731514088560, regionLocation=f8f85b0bb1d8,43467,1731514088560, openSeqNum=2 2024-11-13T16:11:23,062 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97998e7179d85b23a3f5928cd6511629, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=2 2024-11-13T16:11:23,062 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=519ece296a21b5ddccc38a7c928780bb, regionState=OPEN, lastHost=f8f85b0bb1d8,43467,1731514088560, regionLocation=f8f85b0bb1d8,43467,1731514088560, openSeqNum=2 2024-11-13T16:11:23,062 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4a87adf770b37d40ab6691e377ad1b5, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=2 2024-11-13T16:11:23,062 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf1f9f9bc1bc9ffce566d41f8ae3619a, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=2 2024-11-13T16:11:23,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a146f98682f25e8cddcae5dbf318603, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=2 2024-11-13T16:11:23,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6be98d350311c17b11fdb9e9a1238fca, regionState=OPEN, lastHost=f8f85b0bb1d8,43467,1731514088560, regionLocation=f8f85b0bb1d8,43467,1731514088560, openSeqNum=2 2024-11-13T16:11:23,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=72d273770314688e78624993de4f3398, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=2 2024-11-13T16:11:23,063 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-13T16:11:23,063 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731514343063 2024-11-13T16:11:23,063 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731514403063 2024-11-13T16:11:23,063 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-13T16:11:23,063 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure f8f85b0bb1d8,43467,1731514088560, splitWal=true, meta=true, isMeta: false 2024-11-13T16:11:23,066 INFO [PEWorker-1 {}] master.SplitWALManager(105): f8f85b0bb1d8,43467,1731514088560 WAL count=1, meta=false 2024-11-13T16:11:23,067 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856}] 2024-11-13T16:11:23,067 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-13T16:11:23,083 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:11:23,084 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856, worker=f8f85b0bb1d8,41333,1731514088798}] 2024-11-13T16:11:23,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41333 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=63 2024-11-13T16:11:23,264 INFO [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(299): Splitting hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/WALs/f8f85b0bb1d8,43467,1731514088560-splitting/f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856, size=290.7 K (297717bytes) 2024-11-13T16:11:23,264 INFO [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/WALs/f8f85b0bb1d8,43467,1731514088560-splitting/f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856 2024-11-13T16:11:23,265 INFO [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/WALs/f8f85b0bb1d8,43467,1731514088560-splitting/f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856 after 1ms 2024-11-13T16:11:23,280 INFO [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(310): Open hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/WALs/f8f85b0bb1d8,43467,1731514088560-splitting/f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856 took 16ms 2024-11-13T16:11:23,285 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 3945ce860c8642051e17517944a83136: last_flushed_sequence_id: 18446744073709551615 2024-11-13T16:11:23,287 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 6be98d350311c17b11fdb9e9a1238fca: last_flushed_sequence_id: 18446744073709551615 2024-11-13T16:11:23,289 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for aa124b802c708ca1f715cb1f8c9c4eaa: last_flushed_sequence_id: 18446744073709551615 2024-11-13T16:11:23,292 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 29df624bbcb878edce7fcdf0abceea88: last_flushed_sequence_id: 18446744073709551615 2024-11-13T16:11:23,294 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for fec5f21b48c36bf01685ec2ee58f0490: last_flushed_sequence_id: 18446744073709551615 2024-11-13T16:11:23,297 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 519ece296a21b5ddccc38a7c928780bb: last_flushed_sequence_id: 18446744073709551615 2024-11-13T16:11:23,299 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for e8725a794e6d251e1bf6e3502baa7700: last_flushed_sequence_id: 18446744073709551615 2024-11-13T16:11:23,301 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 8ba796d80fc3b65f0948c95ccbdad39d: last_flushed_sequence_id: 18446744073709551615 2024-11-13T16:11:23,309 INFO [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0-Writer-1 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp 2024-11-13T16:11:23,311 INFO [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp 2024-11-13T16:11:23,312 INFO [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0-Writer-1 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3945ce860c8642051e17517944a83136/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp 2024-11-13T16:11:23,314 INFO [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8ba796d80fc3b65f0948c95ccbdad39d/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp 2024-11-13T16:11:23,319 INFO [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp 2024-11-13T16:11:23,322 INFO [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e8725a794e6d251e1bf6e3502baa7700/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp 2024-11-13T16:11:23,322 INFO [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0-Writer-1 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp 2024-11-13T16:11:23,324 INFO [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0-Writer-2 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp 2024-11-13T16:11:23,326 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/WALs/f8f85b0bb1d8,43467,1731514088560-splitting/f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856 so closing down 2024-11-13T16:11:23,326 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T16:11:23,326 INFO [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0-Writer-1 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/29df624bbcb878edce7fcdf0abceea88/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp 2024-11-13T16:11:23,327 INFO [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0-Writer-2 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aa124b802c708ca1f715cb1f8c9c4eaa/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp 2024-11-13T16:11:23,328 INFO [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp 2024-11-13T16:11:23,332 INFO [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0-Writer-2 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp 2024-11-13T16:11:23,333 INFO [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0-Writer-1 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp 2024-11-13T16:11:23,335 INFO [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fec5f21b48c36bf01685ec2ee58f0490/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp 2024-11-13T16:11:23,339 INFO [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0-Writer-2 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6be98d350311c17b11fdb9e9a1238fca/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp 2024-11-13T16:11:23,343 INFO [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0-Writer-1 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/519ece296a21b5ddccc38a7c928780bb/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp 2024-11-13T16:11:23,344 INFO [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T16:11:23,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741869_1045 (size=36767) 2024-11-13T16:11:23,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741869_1045 (size=36767) 2024-11-13T16:11:23,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741869_1045 (size=36767) 2024-11-13T16:11:23,372 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fec5f21b48c36bf01685ec2ee58f0490/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-13T16:11:23,374 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fec5f21b48c36bf01685ec2ee58f0490/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp to hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fec5f21b48c36bf01685ec2ee58f0490/recovered.edits/0000000000000000004 2024-11-13T16:11:23,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741870_1046 (size=36767) 2024-11-13T16:11:23,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741870_1046 (size=36767) 2024-11-13T16:11:23,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741870_1046 (size=36767) 2024-11-13T16:11:23,387 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8ba796d80fc3b65f0948c95ccbdad39d/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp (wrote 1 edits, skipped 0 edits in 1 ms) 2024-11-13T16:11:23,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741871_1047 (size=36767) 2024-11-13T16:11:23,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741871_1047 (size=36767) 2024-11-13T16:11:23,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741871_1047 (size=36767) 2024-11-13T16:11:23,391 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/29df624bbcb878edce7fcdf0abceea88/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp (wrote 1 edits, skipped 0 edits in 1 ms) 2024-11-13T16:11:23,391 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8ba796d80fc3b65f0948c95ccbdad39d/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp to hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8ba796d80fc3b65f0948c95ccbdad39d/recovered.edits/0000000000000000004 2024-11-13T16:11:23,393 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/29df624bbcb878edce7fcdf0abceea88/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp to hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/29df624bbcb878edce7fcdf0abceea88/recovered.edits/0000000000000000004 2024-11-13T16:11:23,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741872_1048 (size=36767) 2024-11-13T16:11:23,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741872_1048 (size=36767) 2024-11-13T16:11:23,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741872_1048 (size=36767) 2024-11-13T16:11:23,413 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3945ce860c8642051e17517944a83136/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp (wrote 1 edits, skipped 0 edits in 2 ms) 2024-11-13T16:11:23,415 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3945ce860c8642051e17517944a83136/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp to hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3945ce860c8642051e17517944a83136/recovered.edits/0000000000000000004 2024-11-13T16:11:23,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741873_1049 (size=36767) 2024-11-13T16:11:23,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741873_1049 (size=36767) 2024-11-13T16:11:23,423 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6be98d350311c17b11fdb9e9a1238fca/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-13T16:11:23,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741873_1049 (size=36767) 2024-11-13T16:11:23,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741874_1050 (size=36767) 2024-11-13T16:11:23,424 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6be98d350311c17b11fdb9e9a1238fca/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp to hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6be98d350311c17b11fdb9e9a1238fca/recovered.edits/0000000000000000004 2024-11-13T16:11:23,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741874_1050 (size=36767) 2024-11-13T16:11:23,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741874_1050 (size=36767) 2024-11-13T16:11:23,428 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aa124b802c708ca1f715cb1f8c9c4eaa/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp (wrote 1 edits, skipped 0 edits in 1 ms) 2024-11-13T16:11:23,430 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aa124b802c708ca1f715cb1f8c9c4eaa/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp to hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aa124b802c708ca1f715cb1f8c9c4eaa/recovered.edits/0000000000000000004 2024-11-13T16:11:23,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741875_1051 (size=36767) 2024-11-13T16:11:23,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741875_1051 (size=36767) 2024-11-13T16:11:23,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741875_1051 (size=36767) 2024-11-13T16:11:23,446 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e8725a794e6d251e1bf6e3502baa7700/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp (wrote 1 edits, skipped 0 edits in 1 ms) 2024-11-13T16:11:23,449 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e8725a794e6d251e1bf6e3502baa7700/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp to hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e8725a794e6d251e1bf6e3502baa7700/recovered.edits/0000000000000000004 2024-11-13T16:11:23,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741876_1052 (size=36767) 2024-11-13T16:11:23,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741876_1052 (size=36767) 2024-11-13T16:11:23,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741876_1052 (size=36767) 2024-11-13T16:11:23,459 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/519ece296a21b5ddccc38a7c928780bb/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-13T16:11:23,461 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/519ece296a21b5ddccc38a7c928780bb/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp to hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/519ece296a21b5ddccc38a7c928780bb/recovered.edits/0000000000000000004 2024-11-13T16:11:23,461 INFO [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(425): Processed 16 edits across 8 Regions in 180 ms; skipped=8; WAL=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/WALs/f8f85b0bb1d8,43467,1731514088560-splitting/f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856, size=290.7 K, length=297717, corrupted=false, cancelled=false 2024-11-13T16:11:23,461 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(428): Completed split of hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/WALs/f8f85b0bb1d8,43467,1731514088560-splitting/f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856, journal: Splitting hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/WALs/f8f85b0bb1d8,43467,1731514088560-splitting/f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856, size=290.7 K (297717bytes) at 1731514283264Creating recovered edits writer path=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3945ce860c8642051e17517944a83136/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp at 1731514283312 (+48 ms)Creating recovered edits writer path=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8ba796d80fc3b65f0948c95ccbdad39d/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp at 1731514283314 (+2 ms)Creating recovered edits writer path=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e8725a794e6d251e1bf6e3502baa7700/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp at 1731514283322 (+8 ms)Finishing writing output for hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/WALs/f8f85b0bb1d8,43467,1731514088560-splitting/f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856 so closing down at 1731514283326 (+4 ms)Creating recovered edits writer path=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/29df624bbcb878edce7fcdf0abceea88/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp at 1731514283326Creating recovered edits writer path=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aa124b802c708ca1f715cb1f8c9c4eaa/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp at 1731514283327 (+1 ms)Creating recovered edits writer path=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fec5f21b48c36bf01685ec2ee58f0490/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp at 1731514283335 (+8 ms)Creating recovered edits writer path=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6be98d350311c17b11fdb9e9a1238fca/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp at 1731514283339 (+4 ms)Creating recovered edits writer path=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/519ece296a21b5ddccc38a7c928780bb/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp at 1731514283343 (+4 ms)3 split writer threads finished at 1731514283344 (+1 ms)Closed recovered edits writer path=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fec5f21b48c36bf01685ec2ee58f0490/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731514283372 (+28 ms)Rename recovered edits hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fec5f21b48c36bf01685ec2ee58f0490/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp to hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fec5f21b48c36bf01685ec2ee58f0490/recovered.edits/0000000000000000004 at 1731514283374 (+2 ms)Closed recovered edits writer path=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8ba796d80fc3b65f0948c95ccbdad39d/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp (wrote 1 edits, skipped 0 edits in 1 ms) at 1731514283387 (+13 ms)Closed recovered edits writer path=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/29df624bbcb878edce7fcdf0abceea88/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp (wrote 1 edits, skipped 0 edits in 1 ms) at 1731514283391 (+4 ms)Rename recovered edits hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8ba796d80fc3b65f0948c95ccbdad39d/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp to hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8ba796d80fc3b65f0948c95ccbdad39d/recovered.edits/0000000000000000004 at 1731514283391Rename recovered edits hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/29df624bbcb878edce7fcdf0abceea88/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp to hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/29df624bbcb878edce7fcdf0abceea88/recovered.edits/0000000000000000004 at 1731514283393 (+2 ms)Closed recovered edits writer path=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3945ce860c8642051e17517944a83136/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp (wrote 1 edits, skipped 0 edits in 2 ms) at 1731514283413 (+20 ms)Rename recovered edits hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3945ce860c8642051e17517944a83136/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp to hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3945ce860c8642051e17517944a83136/recovered.edits/0000000000000000004 at 1731514283415 (+2 ms)Closed recovered edits writer path=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6be98d350311c17b11fdb9e9a1238fca/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731514283423 (+8 ms)Rename recovered edits hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6be98d350311c17b11fdb9e9a1238fca/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp to hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6be98d350311c17b11fdb9e9a1238fca/recovered.edits/0000000000000000004 at 1731514283425 (+2 ms)Closed recovered edits writer path=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aa124b802c708ca1f715cb1f8c9c4eaa/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp (wrote 1 edits, skipped 0 edits in 1 ms) at 1731514283428 (+3 ms)Rename recovered edits hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aa124b802c708ca1f715cb1f8c9c4eaa/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp to hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aa124b802c708ca1f715cb1f8c9c4eaa/recovered.edits/0000000000000000004 at 1731514283430 (+2 ms)Closed recovered edits writer path=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e8725a794e6d251e1bf6e3502baa7700/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp (wrote 1 edits, skipped 0 edits in 1 ms) at 1731514283446 (+16 ms)Rename recovered edits hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e8725a794e6d251e1bf6e3502baa7700/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp to hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e8725a794e6d251e1bf6e3502baa7700/recovered.edits/0000000000000000004 at 1731514283449 (+3 ms)Closed recovered edits writer path=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/519ece296a21b5ddccc38a7c928780bb/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731514283459 (+10 ms)Rename recovered edits hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/519ece296a21b5ddccc38a7c928780bb/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856.temp to hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/519ece296a21b5ddccc38a7c928780bb/recovered.edits/0000000000000000004 at 1731514283461 (+2 ms)Processed 16 edits across 8 Regions in 180 ms; skipped=8; WAL=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/WALs/f8f85b0bb1d8,43467,1731514088560-splitting/f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856, size=290.7 K, length=297717, corrupted=false, cancelled=false at 1731514283461 2024-11-13T16:11:23,462 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/WALs/f8f85b0bb1d8,43467,1731514088560-splitting/f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856 2024-11-13T16:11:23,462 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-13T16:11:23,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42949 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-13T16:11:23,468 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/WALs/f8f85b0bb1d8,43467,1731514088560-splitting/f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856 to hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/oldWALs 2024-11-13T16:11:23,471 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-13T16:11:23,471 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856, worker=f8f85b0bb1d8,41333,1731514088798 in 384 msec 2024-11-13T16:11:23,477 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:11:23,481 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=57 2024-11-13T16:11:23,481 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856, worker=f8f85b0bb1d8,41333,1731514088798 in 410 msec 2024-11-13T16:11:23,485 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/WALs/f8f85b0bb1d8,43467,1731514088560-splitting dir is empty, no logs to split. 2024-11-13T16:11:23,485 INFO [PEWorker-1 {}] master.SplitWALManager(105): f8f85b0bb1d8,43467,1731514088560 WAL count=0, meta=false 2024-11-13T16:11:23,485 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if f8f85b0bb1d8,43467,1731514088560 WAL splitting is done? wals=0, meta=false 2024-11-13T16:11:23,486 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-13T16:11:23,486 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN, hasLock=true; ServerCrashProcedure f8f85b0bb1d8,43467,1731514088560, splitWal=true, meta=true 2024-11-13T16:11:23,486 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T16:11:23,487 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN, hasLock=true; ServerCrashProcedure f8f85b0bb1d8,43467,1731514088560, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T16:11:23,565 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 6 exec state=pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN, hasLock=true; ServerCrashProcedure f8f85b0bb1d8,43467,1731514088560, splitWal=true, meta=true 2024-11-13T16:11:23,565 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-13T16:11:23,565 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T16:11:23,565 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T16:11:23,565 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T16:11:23,566 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-13T16:11:23,566 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-13T16:11:23,566 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-13T16:11:23,566 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T16:11:23,567 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-13T16:11:23,568 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-13T16:11:23,568 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T16:11:23,568 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T16:11:23,568 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T16:11:23,568 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T16:11:23,568 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T16:11:23,568 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T16:11:23,569 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T16:11:23,569 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-13T16:11:23,570 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_DELETE_SPLIT_WALS_DIR, hasLock=false; ServerCrashProcedure f8f85b0bb1d8,43467,1731514088560, splitWal=true, meta=true 2024-11-13T16:11:23,570 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure f8f85b0bb1d8%2C43467%2C1731514088560.meta.1731514091182.meta, worker=f8f85b0bb1d8,38187,1731514088725 2024-11-13T16:11:23,570 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f8f85b0bb1d8%2C43467%2C1731514088560.meta.1731514091182.meta, worker=f8f85b0bb1d8,38187,1731514088725 2024-11-13T16:11:23,570 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T16:11:23,570 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:11:23,570 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856, worker=f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:11:23,570 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856, worker=f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:11:23,571 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_DELETE_SPLIT_WALS_DIR, hasLock=false; ServerCrashProcedure f8f85b0bb1d8,43467,1731514088560, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-13T16:11:23,571 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure f8f85b0bb1d8%2C43467%2C1731514088560.meta.1731514091182.meta, worker=f8f85b0bb1d8,38187,1731514088725 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T16:11:23,571 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f8f85b0bb1d8%2C43467%2C1731514088560.meta.1731514091182.meta, worker=f8f85b0bb1d8,38187,1731514088725 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T16:11:23,571 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T16:11:23,571 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f8f85b0bb1d8,41333,1731514088798 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T16:11:23,571 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856, worker=f8f85b0bb1d8,41333,1731514088798 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T16:11:23,571 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856, worker=f8f85b0bb1d8,41333,1731514088798 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T16:11:23,571 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-13T16:11:23,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=61 2024-11-13T16:11:23,572 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=f8f85b0bb1d8,41333,1731514088798, table=hbase:meta, region=1588230740 2024-11-13T16:11:23,572 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f8f85b0bb1d8,41333,1731514088798, state=OPEN 2024-11-13T16:11:23,577 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42949 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T16:11:23,582 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38187-0x1013621fa650002, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T16:11:23,582 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41333-0x1013621fa650003, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T16:11:23,582 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42949-0x1013621fa650000, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T16:11:23,582 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:11:23,582 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:11:23,582 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:11:23,583 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731514313583 2024-11-13T16:11:23,583 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-13T16:11:23,590 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4aaf5667f132c289386d1b3b0c0ac57f, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=2 2024-11-13T16:11:23,590 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ba796d80fc3b65f0948c95ccbdad39d, regionState=OPEN, lastHost=f8f85b0bb1d8,43467,1731514088560, regionLocation=f8f85b0bb1d8,43467,1731514088560, openSeqNum=2 2024-11-13T16:11:23,590 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93f8ed38b68bd013eb0cf0e7615013ae, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=2 2024-11-13T16:11:23,590 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e72e57b96ff9bc0bbd862ce564d79c42, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=2 2024-11-13T16:11:23,590 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34f3d3b6a6dae1f4077567bed0bd8e75, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=2 2024-11-13T16:11:23,590 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3945ce860c8642051e17517944a83136, regionState=OPEN, lastHost=f8f85b0bb1d8,43467,1731514088560, regionLocation=f8f85b0bb1d8,43467,1731514088560, openSeqNum=2 2024-11-13T16:11:23,591 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1fcf78578ccab75f1705f3c2d7bcdb14, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=2 2024-11-13T16:11:23,591 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a610c2403a0b491d368473a6e1fb049, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=2 2024-11-13T16:11:23,591 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8725a794e6d251e1bf6e3502baa7700, regionState=OPEN, lastHost=f8f85b0bb1d8,43467,1731514088560, regionLocation=f8f85b0bb1d8,43467,1731514088560, openSeqNum=2 2024-11-13T16:11:23,591 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa6636be3559bc84793534420cd4f207, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=2 2024-11-13T16:11:23,591 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=05d65f4a6bfecb3d530d15832c218252, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=2 2024-11-13T16:11:23,591 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78285ca993cc7fbfd78041c30558f756, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=2 2024-11-13T16:11:23,591 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa124b802c708ca1f715cb1f8c9c4eaa, regionState=OPEN, lastHost=f8f85b0bb1d8,43467,1731514088560, regionLocation=f8f85b0bb1d8,43467,1731514088560, openSeqNum=2 2024-11-13T16:11:23,591 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3647edbd8aef23dda3887b61277da1d3, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=2 2024-11-13T16:11:23,592 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=29df624bbcb878edce7fcdf0abceea88, regionState=OPEN, lastHost=f8f85b0bb1d8,43467,1731514088560, regionLocation=f8f85b0bb1d8,43467,1731514088560, openSeqNum=2 2024-11-13T16:11:23,592 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fe97fe0e9b37811ba99ca8100b461589, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=2 2024-11-13T16:11:23,592 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7c34e9ded13254b91545186c85befbf8, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=2 2024-11-13T16:11:23,592 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8f8b8e4ae8470fd8a1d2bf0684947ef, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=2 2024-11-13T16:11:23,592 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fec5f21b48c36bf01685ec2ee58f0490, regionState=OPEN, lastHost=f8f85b0bb1d8,43467,1731514088560, regionLocation=f8f85b0bb1d8,43467,1731514088560, openSeqNum=2 2024-11-13T16:11:23,592 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97998e7179d85b23a3f5928cd6511629, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=2 2024-11-13T16:11:23,592 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=519ece296a21b5ddccc38a7c928780bb, regionState=OPEN, lastHost=f8f85b0bb1d8,43467,1731514088560, regionLocation=f8f85b0bb1d8,43467,1731514088560, openSeqNum=2 2024-11-13T16:11:23,592 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4a87adf770b37d40ab6691e377ad1b5, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=2 2024-11-13T16:11:23,592 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf1f9f9bc1bc9ffce566d41f8ae3619a, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=2 2024-11-13T16:11:23,592 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a146f98682f25e8cddcae5dbf318603, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=2 2024-11-13T16:11:23,593 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6be98d350311c17b11fdb9e9a1238fca, regionState=OPEN, lastHost=f8f85b0bb1d8,43467,1731514088560, regionLocation=f8f85b0bb1d8,43467,1731514088560, openSeqNum=2 2024-11-13T16:11:23,593 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=72d273770314688e78624993de4f3398, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=2 2024-11-13T16:11:23,593 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-13T16:11:23,593 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731514343593 2024-11-13T16:11:23,593 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731514403593 2024-11-13T16:11:23,593 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-13T16:11:23,596 INFO [PEWorker-1 {}] master.MasterWalManager(331): Log dir for server f8f85b0bb1d8,43467,1731514088560 does not exist 2024-11-13T16:11:23,596 INFO [PEWorker-1 {}] master.SplitWALManager(105): f8f85b0bb1d8,43467,1731514088560 WAL count=0, meta=false 2024-11-13T16:11:23,596 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if f8f85b0bb1d8,43467,1731514088560 WAL splitting is done? wals=0, meta=false 2024-11-13T16:11:23,598 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for f8f85b0bb1d8,43467,1731514088560 failed, ignore...File hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/WALs/f8f85b0bb1d8,43467,1731514088560-splitting does not exist. 2024-11-13T16:11:23,598 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-13T16:11:23,599 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=aa124b802c708ca1f715cb1f8c9c4eaa, ASSIGN}, {pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8ba796d80fc3b65f0948c95ccbdad39d, ASSIGN}, {pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e8725a794e6d251e1bf6e3502baa7700, ASSIGN}, {pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3945ce860c8642051e17517944a83136, ASSIGN}, {pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=519ece296a21b5ddccc38a7c928780bb, ASSIGN}, {pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=6be98d350311c17b11fdb9e9a1238fca, ASSIGN}, {pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=29df624bbcb878edce7fcdf0abceea88, ASSIGN}, {pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fec5f21b48c36bf01685ec2ee58f0490, ASSIGN}] 2024-11-13T16:11:23,599 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-13T16:11:23,599 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_FINISH, hasLock=true; ServerCrashProcedure f8f85b0bb1d8,43467,1731514088560, splitWal=true, meta=true 2024-11-13T16:11:23,599 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T16:11:23,599 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_FINISH, hasLock=true; ServerCrashProcedure f8f85b0bb1d8,43467,1731514088560, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T16:11:23,843 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 7 exec state=pid=57, state=WAITING:SERVER_CRASH_FINISH, hasLock=true; ServerCrashProcedure f8f85b0bb1d8,43467,1731514088560, splitWal=true, meta=true 2024-11-13T16:11:23,843 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-13T16:11:23,843 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T16:11:23,843 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T16:11:23,844 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T16:11:23,844 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-13T16:11:23,844 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-13T16:11:23,844 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-13T16:11:23,844 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T16:11:23,845 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-13T16:11:23,847 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-13T16:11:23,847 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T16:11:23,847 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T16:11:23,847 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T16:11:23,847 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T16:11:23,847 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T16:11:23,847 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T16:11:23,848 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T16:11:23,849 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-13T16:11:23,850 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN, hasLock=false; ServerCrashProcedure f8f85b0bb1d8,43467,1731514088560, splitWal=true, meta=true 2024-11-13T16:11:23,850 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure f8f85b0bb1d8%2C43467%2C1731514088560.meta.1731514091182.meta, worker=f8f85b0bb1d8,38187,1731514088725 2024-11-13T16:11:23,850 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f8f85b0bb1d8%2C43467%2C1731514088560.meta.1731514091182.meta, worker=f8f85b0bb1d8,38187,1731514088725 2024-11-13T16:11:23,850 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T16:11:23,851 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:11:23,851 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856, worker=f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:11:23,851 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856, worker=f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:11:23,851 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN, hasLock=false; ServerCrashProcedure f8f85b0bb1d8,43467,1731514088560, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-13T16:11:23,851 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure f8f85b0bb1d8%2C43467%2C1731514088560.meta.1731514091182.meta, worker=f8f85b0bb1d8,38187,1731514088725 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T16:11:23,851 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f8f85b0bb1d8%2C43467%2C1731514088560.meta.1731514091182.meta, worker=f8f85b0bb1d8,38187,1731514088725 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T16:11:23,851 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T16:11:23,851 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f8f85b0bb1d8,41333,1731514088798 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T16:11:23,851 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856, worker=f8f85b0bb1d8,41333,1731514088798 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T16:11:23,851 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856, worker=f8f85b0bb1d8,41333,1731514088798 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T16:11:23,851 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-13T16:11:23,852 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=61 2024-11-13T16:11:23,852 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=f8f85b0bb1d8,41333,1731514088798, table=hbase:meta, region=1588230740 2024-11-13T16:11:23,852 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f8f85b0bb1d8,41333,1731514088798, state=OPEN 2024-11-13T16:11:23,859 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41333-0x1013621fa650003, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T16:11:23,859 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42949-0x1013621fa650000, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T16:11:23,859 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38187-0x1013621fa650002, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T16:11:23,859 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:11:23,859 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:11:23,859 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:11:23,862 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731514313862 2024-11-13T16:11:23,862 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-13T16:11:23,868 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4aaf5667f132c289386d1b3b0c0ac57f, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=2 2024-11-13T16:11:23,868 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ba796d80fc3b65f0948c95ccbdad39d, regionState=OPEN, lastHost=f8f85b0bb1d8,43467,1731514088560, regionLocation=f8f85b0bb1d8,43467,1731514088560, openSeqNum=2 2024-11-13T16:11:23,868 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93f8ed38b68bd013eb0cf0e7615013ae, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=2 2024-11-13T16:11:23,868 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e72e57b96ff9bc0bbd862ce564d79c42, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=2 2024-11-13T16:11:23,868 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34f3d3b6a6dae1f4077567bed0bd8e75, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=2 2024-11-13T16:11:23,869 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3945ce860c8642051e17517944a83136, regionState=OPEN, lastHost=f8f85b0bb1d8,43467,1731514088560, regionLocation=f8f85b0bb1d8,43467,1731514088560, openSeqNum=2 2024-11-13T16:11:23,869 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1fcf78578ccab75f1705f3c2d7bcdb14, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=2 2024-11-13T16:11:23,869 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a610c2403a0b491d368473a6e1fb049, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=2 2024-11-13T16:11:23,869 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8725a794e6d251e1bf6e3502baa7700, regionState=OPEN, lastHost=f8f85b0bb1d8,43467,1731514088560, regionLocation=f8f85b0bb1d8,43467,1731514088560, openSeqNum=2 2024-11-13T16:11:23,869 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa6636be3559bc84793534420cd4f207, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=2 2024-11-13T16:11:23,869 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=05d65f4a6bfecb3d530d15832c218252, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=2 2024-11-13T16:11:23,869 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78285ca993cc7fbfd78041c30558f756, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=2 2024-11-13T16:11:23,869 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa124b802c708ca1f715cb1f8c9c4eaa, regionState=OPEN, lastHost=f8f85b0bb1d8,43467,1731514088560, regionLocation=f8f85b0bb1d8,43467,1731514088560, openSeqNum=2 2024-11-13T16:11:23,869 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3647edbd8aef23dda3887b61277da1d3, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=2 2024-11-13T16:11:23,869 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=29df624bbcb878edce7fcdf0abceea88, regionState=OPEN, lastHost=f8f85b0bb1d8,43467,1731514088560, regionLocation=f8f85b0bb1d8,43467,1731514088560, openSeqNum=2 2024-11-13T16:11:23,870 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fe97fe0e9b37811ba99ca8100b461589, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=2 2024-11-13T16:11:23,870 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7c34e9ded13254b91545186c85befbf8, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=2 2024-11-13T16:11:23,870 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8f8b8e4ae8470fd8a1d2bf0684947ef, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=2 2024-11-13T16:11:23,870 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fec5f21b48c36bf01685ec2ee58f0490, regionState=OPEN, lastHost=f8f85b0bb1d8,43467,1731514088560, regionLocation=f8f85b0bb1d8,43467,1731514088560, openSeqNum=2 2024-11-13T16:11:23,870 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97998e7179d85b23a3f5928cd6511629, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=2 2024-11-13T16:11:23,870 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=519ece296a21b5ddccc38a7c928780bb, regionState=OPEN, lastHost=f8f85b0bb1d8,43467,1731514088560, regionLocation=f8f85b0bb1d8,43467,1731514088560, openSeqNum=2 2024-11-13T16:11:23,870 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4a87adf770b37d40ab6691e377ad1b5, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=2 2024-11-13T16:11:23,870 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf1f9f9bc1bc9ffce566d41f8ae3619a, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=2 2024-11-13T16:11:23,870 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a146f98682f25e8cddcae5dbf318603, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=2 2024-11-13T16:11:23,871 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6be98d350311c17b11fdb9e9a1238fca, regionState=OPEN, lastHost=f8f85b0bb1d8,43467,1731514088560, regionLocation=f8f85b0bb1d8,43467,1731514088560, openSeqNum=2 2024-11-13T16:11:23,871 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=72d273770314688e78624993de4f3398, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=2 2024-11-13T16:11:23,871 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-13T16:11:23,871 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731514343871 2024-11-13T16:11:23,871 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731514403871 2024-11-13T16:11:23,871 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-13T16:11:23,872 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=aa124b802c708ca1f715cb1f8c9c4eaa, ASSIGN}, {pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8ba796d80fc3b65f0948c95ccbdad39d, ASSIGN}, {pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e8725a794e6d251e1bf6e3502baa7700, ASSIGN}, {pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3945ce860c8642051e17517944a83136, ASSIGN}, {pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=519ece296a21b5ddccc38a7c928780bb, ASSIGN}, {pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=6be98d350311c17b11fdb9e9a1238fca, ASSIGN}, {pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=29df624bbcb878edce7fcdf0abceea88, ASSIGN}, {pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fec5f21b48c36bf01685ec2ee58f0490, ASSIGN}] 2024-11-13T16:11:23,872 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-13T16:11:23,875 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fec5f21b48c36bf01685ec2ee58f0490, ASSIGN 2024-11-13T16:11:23,877 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fec5f21b48c36bf01685ec2ee58f0490, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T16:11:23,883 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=29df624bbcb878edce7fcdf0abceea88, ASSIGN 2024-11-13T16:11:23,885 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=29df624bbcb878edce7fcdf0abceea88, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T16:11:23,886 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=6be98d350311c17b11fdb9e9a1238fca, ASSIGN 2024-11-13T16:11:23,891 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=6be98d350311c17b11fdb9e9a1238fca, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T16:11:23,892 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=519ece296a21b5ddccc38a7c928780bb, ASSIGN 2024-11-13T16:11:23,893 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=519ece296a21b5ddccc38a7c928780bb, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T16:11:23,893 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3945ce860c8642051e17517944a83136, ASSIGN 2024-11-13T16:11:23,894 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3945ce860c8642051e17517944a83136, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T16:11:23,895 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e8725a794e6d251e1bf6e3502baa7700, ASSIGN 2024-11-13T16:11:23,896 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e8725a794e6d251e1bf6e3502baa7700, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T16:11:23,897 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8ba796d80fc3b65f0948c95ccbdad39d, ASSIGN 2024-11-13T16:11:23,897 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8ba796d80fc3b65f0948c95ccbdad39d, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T16:11:23,899 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=aa124b802c708ca1f715cb1f8c9c4eaa, ASSIGN 2024-11-13T16:11:23,900 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=aa124b802c708ca1f715cb1f8c9c4eaa, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T16:11:24,028 DEBUG [f8f85b0bb1d8:42949 {}] balancer.BalancerClusterState(204): Hosts are {f8f85b0bb1d8=0} racks are {/default-rack=0} 2024-11-13T16:11:24,028 DEBUG [f8f85b0bb1d8:42949 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T16:11:24,029 DEBUG [f8f85b0bb1d8:42949 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T16:11:24,029 DEBUG [f8f85b0bb1d8:42949 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T16:11:24,029 DEBUG [f8f85b0bb1d8:42949 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T16:11:24,029 INFO [f8f85b0bb1d8:42949 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T16:11:24,029 INFO [f8f85b0bb1d8:42949 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T16:11:24,029 DEBUG [f8f85b0bb1d8:42949 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T16:11:24,029 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=519ece296a21b5ddccc38a7c928780bb, regionState=OPENING, regionLocation=f8f85b0bb1d8,38187,1731514088725 2024-11-13T16:11:24,031 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=29df624bbcb878edce7fcdf0abceea88, regionState=OPENING, regionLocation=f8f85b0bb1d8,38187,1731514088725 2024-11-13T16:11:24,033 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=519ece296a21b5ddccc38a7c928780bb, ASSIGN because future has completed 2024-11-13T16:11:24,034 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=e8725a794e6d251e1bf6e3502baa7700, regionState=OPENING, regionLocation=f8f85b0bb1d8,38187,1731514088725 2024-11-13T16:11:24,036 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=29df624bbcb878edce7fcdf0abceea88, ASSIGN because future has completed 2024-11-13T16:11:24,037 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=72, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 519ece296a21b5ddccc38a7c928780bb, server=f8f85b0bb1d8,38187,1731514088725}] 2024-11-13T16:11:24,038 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e8725a794e6d251e1bf6e3502baa7700, ASSIGN because future has completed 2024-11-13T16:11:24,048 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure e8725a794e6d251e1bf6e3502baa7700, server=f8f85b0bb1d8,38187,1731514088725}] 2024-11-13T16:11:24,052 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 29df624bbcb878edce7fcdf0abceea88, server=f8f85b0bb1d8,38187,1731514088725}] 2024-11-13T16:11:24,060 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=65 updating hbase:meta row=8ba796d80fc3b65f0948c95ccbdad39d, regionState=OPENING, regionLocation=f8f85b0bb1d8,38187,1731514088725 2024-11-13T16:11:24,061 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=6be98d350311c17b11fdb9e9a1238fca, regionState=OPENING, regionLocation=f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:11:24,062 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=fec5f21b48c36bf01685ec2ee58f0490, regionState=OPENING, regionLocation=f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:11:24,063 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=aa124b802c708ca1f715cb1f8c9c4eaa, regionState=OPENING, regionLocation=f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:11:24,064 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=3945ce860c8642051e17517944a83136, regionState=OPENING, regionLocation=f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:11:24,071 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8ba796d80fc3b65f0948c95ccbdad39d, ASSIGN because future has completed 2024-11-13T16:11:24,078 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ba796d80fc3b65f0948c95ccbdad39d, server=f8f85b0bb1d8,38187,1731514088725}] 2024-11-13T16:11:24,079 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=6be98d350311c17b11fdb9e9a1238fca, ASSIGN because future has completed 2024-11-13T16:11:24,084 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fec5f21b48c36bf01685ec2ee58f0490, ASSIGN because future has completed 2024-11-13T16:11:24,104 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=aa124b802c708ca1f715cb1f8c9c4eaa, ASSIGN because future has completed 2024-11-13T16:11:24,112 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3945ce860c8642051e17517944a83136, ASSIGN because future has completed 2024-11-13T16:11:24,127 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3945ce860c8642051e17517944a83136, server=f8f85b0bb1d8,41333,1731514088798}] 2024-11-13T16:11:24,140 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure aa124b802c708ca1f715cb1f8c9c4eaa, server=f8f85b0bb1d8,41333,1731514088798}] 2024-11-13T16:11:24,143 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure fec5f21b48c36bf01685ec2ee58f0490, server=f8f85b0bb1d8,41333,1731514088798}] 2024-11-13T16:11:24,146 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6be98d350311c17b11fdb9e9a1238fca, server=f8f85b0bb1d8,41333,1731514088798}] 2024-11-13T16:11:24,198 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50801, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T16:11:24,204 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731514091776.8ba796d80fc3b65f0948c95ccbdad39d. 2024-11-13T16:11:24,205 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => 8ba796d80fc3b65f0948c95ccbdad39d, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731514091776.8ba796d80fc3b65f0948c95ccbdad39d.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-13T16:11:24,205 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 8ba796d80fc3b65f0948c95ccbdad39d 2024-11-13T16:11:24,205 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731514091776.8ba796d80fc3b65f0948c95ccbdad39d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:11:24,206 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for 8ba796d80fc3b65f0948c95ccbdad39d 2024-11-13T16:11:24,206 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for 8ba796d80fc3b65f0948c95ccbdad39d 2024-11-13T16:11:24,211 INFO [StoreOpener-8ba796d80fc3b65f0948c95ccbdad39d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 8ba796d80fc3b65f0948c95ccbdad39d 2024-11-13T16:11:24,212 INFO [StoreOpener-8ba796d80fc3b65f0948c95ccbdad39d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8ba796d80fc3b65f0948c95ccbdad39d columnFamilyName colfamily11 2024-11-13T16:11:24,212 DEBUG [StoreOpener-8ba796d80fc3b65f0948c95ccbdad39d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:11:24,213 INFO [StoreOpener-8ba796d80fc3b65f0948c95ccbdad39d-1 {}] regionserver.HStore(327): Store=8ba796d80fc3b65f0948c95ccbdad39d/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:11:24,213 INFO [StoreOpener-8ba796d80fc3b65f0948c95ccbdad39d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 8ba796d80fc3b65f0948c95ccbdad39d 2024-11-13T16:11:24,215 INFO [StoreOpener-8ba796d80fc3b65f0948c95ccbdad39d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8ba796d80fc3b65f0948c95ccbdad39d columnFamilyName colfamily21 2024-11-13T16:11:24,215 DEBUG [StoreOpener-8ba796d80fc3b65f0948c95ccbdad39d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:11:24,217 INFO [StoreOpener-8ba796d80fc3b65f0948c95ccbdad39d-1 {}] regionserver.HStore(327): Store=8ba796d80fc3b65f0948c95ccbdad39d/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:11:24,217 INFO [StoreOpener-8ba796d80fc3b65f0948c95ccbdad39d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 8ba796d80fc3b65f0948c95ccbdad39d 2024-11-13T16:11:24,227 INFO [StoreOpener-8ba796d80fc3b65f0948c95ccbdad39d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8ba796d80fc3b65f0948c95ccbdad39d columnFamilyName colfamily31 2024-11-13T16:11:24,227 DEBUG [StoreOpener-8ba796d80fc3b65f0948c95ccbdad39d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:11:24,237 INFO [StoreOpener-8ba796d80fc3b65f0948c95ccbdad39d-1 {}] regionserver.HStore(327): Store=8ba796d80fc3b65f0948c95ccbdad39d/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:11:24,238 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for 8ba796d80fc3b65f0948c95ccbdad39d 2024-11-13T16:11:24,243 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8ba796d80fc3b65f0948c95ccbdad39d 2024-11-13T16:11:24,245 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8ba796d80fc3b65f0948c95ccbdad39d 2024-11-13T16:11:24,246 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8ba796d80fc3b65f0948c95ccbdad39d/recovered.edits/0000000000000000004 2024-11-13T16:11:24,256 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8ba796d80fc3b65f0948c95ccbdad39d/recovered.edits/0000000000000000004 2024-11-13T16:11:24,256 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(2902): Flushing 8ba796d80fc3b65f0948c95ccbdad39d 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T16:11:24,288 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731514091776.3945ce860c8642051e17517944a83136. 2024-11-13T16:11:24,288 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => 3945ce860c8642051e17517944a83136, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731514091776.3945ce860c8642051e17517944a83136.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-13T16:11:24,289 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 3945ce860c8642051e17517944a83136 2024-11-13T16:11:24,289 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731514091776.3945ce860c8642051e17517944a83136.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:11:24,289 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for 3945ce860c8642051e17517944a83136 2024-11-13T16:11:24,289 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for 3945ce860c8642051e17517944a83136 2024-11-13T16:11:24,295 INFO [StoreOpener-3945ce860c8642051e17517944a83136-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 3945ce860c8642051e17517944a83136 2024-11-13T16:11:24,297 INFO [StoreOpener-3945ce860c8642051e17517944a83136-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3945ce860c8642051e17517944a83136 columnFamilyName colfamily11 2024-11-13T16:11:24,297 DEBUG [StoreOpener-3945ce860c8642051e17517944a83136-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:11:24,298 INFO [StoreOpener-3945ce860c8642051e17517944a83136-1 {}] regionserver.HStore(327): Store=3945ce860c8642051e17517944a83136/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:11:24,298 INFO [StoreOpener-3945ce860c8642051e17517944a83136-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 3945ce860c8642051e17517944a83136 2024-11-13T16:11:24,301 INFO [StoreOpener-3945ce860c8642051e17517944a83136-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3945ce860c8642051e17517944a83136 columnFamilyName colfamily21 2024-11-13T16:11:24,301 DEBUG [StoreOpener-3945ce860c8642051e17517944a83136-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:11:24,304 INFO [StoreOpener-3945ce860c8642051e17517944a83136-1 {}] regionserver.HStore(327): Store=3945ce860c8642051e17517944a83136/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:11:24,304 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8ba796d80fc3b65f0948c95ccbdad39d/.tmp/colfamily11/2b9999bdebaf4fc4bfeb011d9fb4d53e is 52, key is bbb/colfamily11:colfamily11/1731514098196/Put/seqid=0 2024-11-13T16:11:24,311 INFO [StoreOpener-3945ce860c8642051e17517944a83136-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 3945ce860c8642051e17517944a83136 2024-11-13T16:11:24,314 INFO [StoreOpener-3945ce860c8642051e17517944a83136-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3945ce860c8642051e17517944a83136 columnFamilyName colfamily31 2024-11-13T16:11:24,314 DEBUG [StoreOpener-3945ce860c8642051e17517944a83136-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:11:24,318 INFO [StoreOpener-3945ce860c8642051e17517944a83136-1 {}] regionserver.HStore(327): Store=3945ce860c8642051e17517944a83136/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:11:24,318 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for 3945ce860c8642051e17517944a83136 2024-11-13T16:11:24,320 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3945ce860c8642051e17517944a83136 2024-11-13T16:11:24,335 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3945ce860c8642051e17517944a83136 2024-11-13T16:11:24,344 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3945ce860c8642051e17517944a83136/recovered.edits/0000000000000000004 2024-11-13T16:11:24,366 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3945ce860c8642051e17517944a83136/recovered.edits/0000000000000000004 2024-11-13T16:11:24,366 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(2902): Flushing 3945ce860c8642051e17517944a83136 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T16:11:24,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741877_1053 (size=39263) 2024-11-13T16:11:24,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741877_1053 (size=39263) 2024-11-13T16:11:24,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741877_1053 (size=39263) 2024-11-13T16:11:24,407 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8ba796d80fc3b65f0948c95ccbdad39d/.tmp/colfamily11/2b9999bdebaf4fc4bfeb011d9fb4d53e 2024-11-13T16:11:24,413 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3945ce860c8642051e17517944a83136/.tmp/colfamily11/9964ad32a7b54de28f749b1af789cf3e is 52, key is fff/colfamily11:colfamily11/1731514098285/Put/seqid=0 2024-11-13T16:11:24,475 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8ba796d80fc3b65f0948c95ccbdad39d/.tmp/colfamily11/2b9999bdebaf4fc4bfeb011d9fb4d53e as hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8ba796d80fc3b65f0948c95ccbdad39d/colfamily11/2b9999bdebaf4fc4bfeb011d9fb4d53e 2024-11-13T16:11:24,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741878_1054 (size=39263) 2024-11-13T16:11:24,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741878_1054 (size=39263) 2024-11-13T16:11:24,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741878_1054 (size=39263) 2024-11-13T16:11:24,508 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3945ce860c8642051e17517944a83136/.tmp/colfamily11/9964ad32a7b54de28f749b1af789cf3e 2024-11-13T16:11:24,516 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8ba796d80fc3b65f0948c95ccbdad39d/colfamily11/2b9999bdebaf4fc4bfeb011d9fb4d53e, entries=703, sequenceid=4, filesize=38.3 K 2024-11-13T16:11:24,516 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 8ba796d80fc3b65f0948c95ccbdad39d in 260ms, sequenceid=4, compaction requested=false; wal=null 2024-11-13T16:11:24,520 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8ba796d80fc3b65f0948c95ccbdad39d/recovered.edits/0000000000000000004 2024-11-13T16:11:24,525 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for 8ba796d80fc3b65f0948c95ccbdad39d 2024-11-13T16:11:24,525 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for 8ba796d80fc3b65f0948c95ccbdad39d 2024-11-13T16:11:24,528 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T16:11:24,538 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for 8ba796d80fc3b65f0948c95ccbdad39d 2024-11-13T16:11:24,559 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8ba796d80fc3b65f0948c95ccbdad39d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T16:11:24,567 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened 8ba796d80fc3b65f0948c95ccbdad39d; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64042895, jitterRate=-0.045686498284339905}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T16:11:24,567 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8ba796d80fc3b65f0948c95ccbdad39d 2024-11-13T16:11:24,567 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for 8ba796d80fc3b65f0948c95ccbdad39d: Running coprocessor pre-open hook at 1731514284206Writing region info on filesystem at 1731514284206Initializing all the Stores at 1731514284207 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514284207Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514284210 (+3 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514284211 (+1 ms)Obtaining lock to block concurrent updates at 1731514284256 (+45 ms)Preparing flush snapshotting stores in 8ba796d80fc3b65f0948c95ccbdad39d at 1731514284256Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731514091776.8ba796d80fc3b65f0948c95ccbdad39d., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731514284256Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731514091776.8ba796d80fc3b65f0948c95ccbdad39d. at 1731514284256Flushing 8ba796d80fc3b65f0948c95ccbdad39d/colfamily11: creating writer at 1731514284257 (+1 ms)Flushing 8ba796d80fc3b65f0948c95ccbdad39d/colfamily11: appending metadata at 1731514284303 (+46 ms)Flushing 8ba796d80fc3b65f0948c95ccbdad39d/colfamily11: closing flushed file at 1731514284303Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@768e63b2: reopening flushed file at 1731514284463 (+160 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 8ba796d80fc3b65f0948c95ccbdad39d in 260ms, sequenceid=4, compaction requested=false; wal=null at 1731514284516 (+53 ms)Cleaning up temporary data from old regions at 1731514284525 (+9 ms)Running coprocessor post-open hooks at 1731514284567 (+42 ms)Region opened successfully at 1731514284567 2024-11-13T16:11:24,579 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3945ce860c8642051e17517944a83136/.tmp/colfamily11/9964ad32a7b54de28f749b1af789cf3e as hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3945ce860c8642051e17517944a83136/colfamily11/9964ad32a7b54de28f749b1af789cf3e 2024-11-13T16:11:24,579 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731514091776.8ba796d80fc3b65f0948c95ccbdad39d., pid=75, masterSystemTime=1731514284196 2024-11-13T16:11:24,594 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731514091776.8ba796d80fc3b65f0948c95ccbdad39d. 2024-11-13T16:11:24,594 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731514091776.8ba796d80fc3b65f0948c95ccbdad39d. 2024-11-13T16:11:24,594 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731514091776.519ece296a21b5ddccc38a7c928780bb. 2024-11-13T16:11:24,594 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7752): Opening region: {ENCODED => 519ece296a21b5ddccc38a7c928780bb, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731514091776.519ece296a21b5ddccc38a7c928780bb.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-13T16:11:24,595 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 519ece296a21b5ddccc38a7c928780bb 2024-11-13T16:11:24,595 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731514091776.519ece296a21b5ddccc38a7c928780bb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:11:24,595 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7794): checking encryption for 519ece296a21b5ddccc38a7c928780bb 2024-11-13T16:11:24,595 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7797): checking classloading for 519ece296a21b5ddccc38a7c928780bb 2024-11-13T16:11:24,599 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=65 updating hbase:meta row=8ba796d80fc3b65f0948c95ccbdad39d, regionState=OPEN, openSeqNum=5, regionLocation=f8f85b0bb1d8,38187,1731514088725 2024-11-13T16:11:24,607 INFO [StoreOpener-519ece296a21b5ddccc38a7c928780bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 519ece296a21b5ddccc38a7c928780bb 2024-11-13T16:11:24,611 INFO [StoreOpener-519ece296a21b5ddccc38a7c928780bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 519ece296a21b5ddccc38a7c928780bb columnFamilyName colfamily11 2024-11-13T16:11:24,611 DEBUG [StoreOpener-519ece296a21b5ddccc38a7c928780bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:11:24,612 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ba796d80fc3b65f0948c95ccbdad39d, server=f8f85b0bb1d8,38187,1731514088725 because future has completed 2024-11-13T16:11:24,614 INFO [StoreOpener-519ece296a21b5ddccc38a7c928780bb-1 {}] regionserver.HStore(327): Store=519ece296a21b5ddccc38a7c928780bb/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:11:24,614 INFO [StoreOpener-519ece296a21b5ddccc38a7c928780bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 519ece296a21b5ddccc38a7c928780bb 2024-11-13T16:11:24,614 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42949 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=f8f85b0bb1d8,38187,1731514088725, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8ba796d80fc3b65f0948c95ccbdad39d. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T16:11:24,619 INFO [StoreOpener-519ece296a21b5ddccc38a7c928780bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 519ece296a21b5ddccc38a7c928780bb columnFamilyName colfamily21 2024-11-13T16:11:24,619 DEBUG [StoreOpener-519ece296a21b5ddccc38a7c928780bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:11:24,620 INFO [StoreOpener-519ece296a21b5ddccc38a7c928780bb-1 {}] regionserver.HStore(327): Store=519ece296a21b5ddccc38a7c928780bb/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:11:24,620 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3945ce860c8642051e17517944a83136/colfamily11/9964ad32a7b54de28f749b1af789cf3e, entries=703, sequenceid=4, filesize=38.3 K 2024-11-13T16:11:24,620 INFO [StoreOpener-519ece296a21b5ddccc38a7c928780bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 519ece296a21b5ddccc38a7c928780bb 2024-11-13T16:11:24,621 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 3945ce860c8642051e17517944a83136 in 255ms, sequenceid=4, compaction requested=false; wal=null 2024-11-13T16:11:24,626 INFO [StoreOpener-519ece296a21b5ddccc38a7c928780bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 519ece296a21b5ddccc38a7c928780bb columnFamilyName colfamily31 2024-11-13T16:11:24,626 DEBUG [StoreOpener-519ece296a21b5ddccc38a7c928780bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:11:24,626 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3945ce860c8642051e17517944a83136/recovered.edits/0000000000000000004 2024-11-13T16:11:24,631 INFO [StoreOpener-519ece296a21b5ddccc38a7c928780bb-1 {}] regionserver.HStore(327): Store=519ece296a21b5ddccc38a7c928780bb/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:11:24,631 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1038): replaying wal for 519ece296a21b5ddccc38a7c928780bb 2024-11-13T16:11:24,635 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=65 2024-11-13T16:11:24,635 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=65, state=SUCCESS, hasLock=false; OpenRegionProcedure 8ba796d80fc3b65f0948c95ccbdad39d, server=f8f85b0bb1d8,38187,1731514088725 in 538 msec 2024-11-13T16:11:24,637 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/519ece296a21b5ddccc38a7c928780bb 2024-11-13T16:11:24,643 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8ba796d80fc3b65f0948c95ccbdad39d, ASSIGN in 765 msec 2024-11-13T16:11:24,646 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for 3945ce860c8642051e17517944a83136 2024-11-13T16:11:24,646 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for 3945ce860c8642051e17517944a83136 2024-11-13T16:11:24,651 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T16:11:24,651 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/519ece296a21b5ddccc38a7c928780bb 2024-11-13T16:11:24,661 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/519ece296a21b5ddccc38a7c928780bb/recovered.edits/0000000000000000004 2024-11-13T16:11:24,662 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for 3945ce860c8642051e17517944a83136 2024-11-13T16:11:24,673 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/519ece296a21b5ddccc38a7c928780bb/recovered.edits/0000000000000000004 2024-11-13T16:11:24,674 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(2902): Flushing 519ece296a21b5ddccc38a7c928780bb 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T16:11:24,679 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3945ce860c8642051e17517944a83136/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T16:11:24,691 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened 3945ce860c8642051e17517944a83136; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59101982, jitterRate=-0.11931183934211731}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T16:11:24,691 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3945ce860c8642051e17517944a83136 2024-11-13T16:11:24,691 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for 3945ce860c8642051e17517944a83136: Running coprocessor pre-open hook at 1731514284289Writing region info on filesystem at 1731514284289Initializing all the Stores at 1731514284291 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514284291Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514284294 (+3 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514284295 (+1 ms)Obtaining lock to block concurrent updates at 1731514284366 (+71 ms)Preparing flush snapshotting stores in 3945ce860c8642051e17517944a83136 at 1731514284366Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731514091776.3945ce860c8642051e17517944a83136., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731514284366Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731514091776.3945ce860c8642051e17517944a83136. at 1731514284366Flushing 3945ce860c8642051e17517944a83136/colfamily11: creating writer at 1731514284366Flushing 3945ce860c8642051e17517944a83136/colfamily11: appending metadata at 1731514284413 (+47 ms)Flushing 3945ce860c8642051e17517944a83136/colfamily11: closing flushed file at 1731514284413Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@41b72074: reopening flushed file at 1731514284559 (+146 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 3945ce860c8642051e17517944a83136 in 255ms, sequenceid=4, compaction requested=false; wal=null at 1731514284621 (+62 ms)Cleaning up temporary data from old regions at 1731514284646 (+25 ms)Running coprocessor post-open hooks at 1731514284691 (+45 ms)Region opened successfully at 1731514284691 2024-11-13T16:11:24,697 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731514091776.3945ce860c8642051e17517944a83136., pid=76, masterSystemTime=1731514284282 2024-11-13T16:11:24,705 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731514091776.3945ce860c8642051e17517944a83136. 2024-11-13T16:11:24,705 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731514091776.3945ce860c8642051e17517944a83136. 2024-11-13T16:11:24,705 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731514091776.6be98d350311c17b11fdb9e9a1238fca. 2024-11-13T16:11:24,705 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => 6be98d350311c17b11fdb9e9a1238fca, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731514091776.6be98d350311c17b11fdb9e9a1238fca.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-13T16:11:24,706 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 6be98d350311c17b11fdb9e9a1238fca 2024-11-13T16:11:24,706 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731514091776.6be98d350311c17b11fdb9e9a1238fca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:11:24,706 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for 6be98d350311c17b11fdb9e9a1238fca 2024-11-13T16:11:24,706 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for 6be98d350311c17b11fdb9e9a1238fca 2024-11-13T16:11:24,708 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=3945ce860c8642051e17517944a83136, regionState=OPEN, openSeqNum=5, regionLocation=f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:11:24,714 INFO [StoreOpener-6be98d350311c17b11fdb9e9a1238fca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 6be98d350311c17b11fdb9e9a1238fca 2024-11-13T16:11:24,720 INFO [StoreOpener-6be98d350311c17b11fdb9e9a1238fca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6be98d350311c17b11fdb9e9a1238fca columnFamilyName colfamily11 2024-11-13T16:11:24,720 DEBUG [StoreOpener-6be98d350311c17b11fdb9e9a1238fca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:11:24,725 INFO [StoreOpener-6be98d350311c17b11fdb9e9a1238fca-1 {}] regionserver.HStore(327): Store=6be98d350311c17b11fdb9e9a1238fca/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:11:24,726 INFO [StoreOpener-6be98d350311c17b11fdb9e9a1238fca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 6be98d350311c17b11fdb9e9a1238fca 2024-11-13T16:11:24,727 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3945ce860c8642051e17517944a83136, server=f8f85b0bb1d8,41333,1731514088798 because future has completed 2024-11-13T16:11:24,731 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/519ece296a21b5ddccc38a7c928780bb/.tmp/colfamily11/b8e49916fad04fc08a42eeca8b0c0a9c is 52, key is uuu/colfamily11:colfamily11/1731514098503/Put/seqid=0 2024-11-13T16:11:24,732 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42949 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=f8f85b0bb1d8,41333,1731514088798, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3945ce860c8642051e17517944a83136. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T16:11:24,735 INFO [StoreOpener-6be98d350311c17b11fdb9e9a1238fca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6be98d350311c17b11fdb9e9a1238fca columnFamilyName colfamily21 2024-11-13T16:11:24,735 DEBUG [StoreOpener-6be98d350311c17b11fdb9e9a1238fca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:11:24,743 INFO [StoreOpener-6be98d350311c17b11fdb9e9a1238fca-1 {}] regionserver.HStore(327): Store=6be98d350311c17b11fdb9e9a1238fca/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:11:24,743 INFO [StoreOpener-6be98d350311c17b11fdb9e9a1238fca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 6be98d350311c17b11fdb9e9a1238fca 2024-11-13T16:11:24,747 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=76, resume processing ppid=67 2024-11-13T16:11:24,747 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, ppid=67, state=SUCCESS, hasLock=false; OpenRegionProcedure 3945ce860c8642051e17517944a83136, server=f8f85b0bb1d8,41333,1731514088798 in 608 msec 2024-11-13T16:11:24,755 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3945ce860c8642051e17517944a83136, ASSIGN in 876 msec 2024-11-13T16:11:24,755 INFO [StoreOpener-6be98d350311c17b11fdb9e9a1238fca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6be98d350311c17b11fdb9e9a1238fca columnFamilyName colfamily31 2024-11-13T16:11:24,755 DEBUG [StoreOpener-6be98d350311c17b11fdb9e9a1238fca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:11:24,763 INFO [StoreOpener-6be98d350311c17b11fdb9e9a1238fca-1 {}] regionserver.HStore(327): Store=6be98d350311c17b11fdb9e9a1238fca/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:11:24,763 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for 6be98d350311c17b11fdb9e9a1238fca 2024-11-13T16:11:24,767 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6be98d350311c17b11fdb9e9a1238fca 2024-11-13T16:11:24,792 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6be98d350311c17b11fdb9e9a1238fca 2024-11-13T16:11:24,799 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6be98d350311c17b11fdb9e9a1238fca/recovered.edits/0000000000000000004 2024-11-13T16:11:24,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741879_1055 (size=39263) 2024-11-13T16:11:24,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741879_1055 (size=39263) 2024-11-13T16:11:24,820 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/519ece296a21b5ddccc38a7c928780bb/.tmp/colfamily11/b8e49916fad04fc08a42eeca8b0c0a9c 2024-11-13T16:11:24,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741879_1055 (size=39263) 2024-11-13T16:11:24,823 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6be98d350311c17b11fdb9e9a1238fca/recovered.edits/0000000000000000004 2024-11-13T16:11:24,823 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(2902): Flushing 6be98d350311c17b11fdb9e9a1238fca 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T16:11:24,867 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/519ece296a21b5ddccc38a7c928780bb/.tmp/colfamily11/b8e49916fad04fc08a42eeca8b0c0a9c as hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/519ece296a21b5ddccc38a7c928780bb/colfamily11/b8e49916fad04fc08a42eeca8b0c0a9c 2024-11-13T16:11:24,879 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6be98d350311c17b11fdb9e9a1238fca/.tmp/colfamily11/b229f1a38725438ebe67fb1caf2fcc7e is 52, key is yyy/colfamily11:colfamily11/1731514098542/Put/seqid=0 2024-11-13T16:11:24,894 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/519ece296a21b5ddccc38a7c928780bb/colfamily11/b8e49916fad04fc08a42eeca8b0c0a9c, entries=703, sequenceid=4, filesize=38.3 K 2024-11-13T16:11:24,896 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 519ece296a21b5ddccc38a7c928780bb in 222ms, sequenceid=4, compaction requested=false; wal=null 2024-11-13T16:11:24,907 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/519ece296a21b5ddccc38a7c928780bb/recovered.edits/0000000000000000004 2024-11-13T16:11:24,917 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1048): stopping wal replay for 519ece296a21b5ddccc38a7c928780bb 2024-11-13T16:11:24,917 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1060): Cleaning up temporary data for 519ece296a21b5ddccc38a7c928780bb 2024-11-13T16:11:24,919 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T16:11:24,926 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1093): writing seq id for 519ece296a21b5ddccc38a7c928780bb 2024-11-13T16:11:24,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741880_1056 (size=39263) 2024-11-13T16:11:24,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741880_1056 (size=39263) 2024-11-13T16:11:24,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741880_1056 (size=39263) 2024-11-13T16:11:24,971 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/519ece296a21b5ddccc38a7c928780bb/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T16:11:24,975 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6be98d350311c17b11fdb9e9a1238fca/.tmp/colfamily11/b229f1a38725438ebe67fb1caf2fcc7e 2024-11-13T16:11:24,983 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1114): Opened 519ece296a21b5ddccc38a7c928780bb; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61800436, jitterRate=-0.07910174131393433}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T16:11:24,983 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 519ece296a21b5ddccc38a7c928780bb 2024-11-13T16:11:24,983 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1006): Region open journal for 519ece296a21b5ddccc38a7c928780bb: Running coprocessor pre-open hook at 1731514284596Writing region info on filesystem at 1731514284596Initializing all the Stores at 1731514284603 (+7 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514284603Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514284611 (+8 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514284611Obtaining lock to block concurrent updates at 1731514284674 (+63 ms)Preparing flush snapshotting stores in 519ece296a21b5ddccc38a7c928780bb at 1731514284674Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731514091776.519ece296a21b5ddccc38a7c928780bb., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731514284674Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731514091776.519ece296a21b5ddccc38a7c928780bb. at 1731514284674Flushing 519ece296a21b5ddccc38a7c928780bb/colfamily11: creating writer at 1731514284674Flushing 519ece296a21b5ddccc38a7c928780bb/colfamily11: appending metadata at 1731514284731 (+57 ms)Flushing 519ece296a21b5ddccc38a7c928780bb/colfamily11: closing flushed file at 1731514284731Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7939f1e8: reopening flushed file at 1731514284857 (+126 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 519ece296a21b5ddccc38a7c928780bb in 222ms, sequenceid=4, compaction requested=false; wal=null at 1731514284896 (+39 ms)Cleaning up temporary data from old regions at 1731514284917 (+21 ms)Running coprocessor post-open hooks at 1731514284983 (+66 ms)Region opened successfully at 1731514284983 2024-11-13T16:11:24,986 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731514091776.519ece296a21b5ddccc38a7c928780bb., pid=72, masterSystemTime=1731514284196 2024-11-13T16:11:24,990 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731514091776.519ece296a21b5ddccc38a7c928780bb. 2024-11-13T16:11:24,990 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731514091776.519ece296a21b5ddccc38a7c928780bb. 2024-11-13T16:11:24,991 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731514091776.29df624bbcb878edce7fcdf0abceea88. 2024-11-13T16:11:24,991 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=519ece296a21b5ddccc38a7c928780bb, regionState=OPEN, openSeqNum=5, regionLocation=f8f85b0bb1d8,38187,1731514088725 2024-11-13T16:11:24,992 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7752): Opening region: {ENCODED => 29df624bbcb878edce7fcdf0abceea88, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731514091776.29df624bbcb878edce7fcdf0abceea88.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-13T16:11:24,993 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 29df624bbcb878edce7fcdf0abceea88 2024-11-13T16:11:24,993 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731514091776.29df624bbcb878edce7fcdf0abceea88.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:11:24,993 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7794): checking encryption for 29df624bbcb878edce7fcdf0abceea88 2024-11-13T16:11:24,993 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7797): checking classloading for 29df624bbcb878edce7fcdf0abceea88 2024-11-13T16:11:24,999 INFO [StoreOpener-29df624bbcb878edce7fcdf0abceea88-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 29df624bbcb878edce7fcdf0abceea88 2024-11-13T16:11:25,004 INFO [StoreOpener-29df624bbcb878edce7fcdf0abceea88-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 29df624bbcb878edce7fcdf0abceea88 columnFamilyName colfamily11 2024-11-13T16:11:25,008 DEBUG [StoreOpener-29df624bbcb878edce7fcdf0abceea88-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:11:25,007 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=72, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 519ece296a21b5ddccc38a7c928780bb, server=f8f85b0bb1d8,38187,1731514088725 because future has completed 2024-11-13T16:11:25,011 INFO [StoreOpener-29df624bbcb878edce7fcdf0abceea88-1 {}] regionserver.HStore(327): Store=29df624bbcb878edce7fcdf0abceea88/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:11:25,011 INFO [StoreOpener-29df624bbcb878edce7fcdf0abceea88-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 29df624bbcb878edce7fcdf0abceea88 2024-11-13T16:11:25,014 INFO [StoreOpener-29df624bbcb878edce7fcdf0abceea88-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 29df624bbcb878edce7fcdf0abceea88 columnFamilyName colfamily21 2024-11-13T16:11:25,014 DEBUG [StoreOpener-29df624bbcb878edce7fcdf0abceea88-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:11:25,019 INFO [StoreOpener-29df624bbcb878edce7fcdf0abceea88-1 {}] regionserver.HStore(327): Store=29df624bbcb878edce7fcdf0abceea88/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:11:25,019 INFO [StoreOpener-29df624bbcb878edce7fcdf0abceea88-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 29df624bbcb878edce7fcdf0abceea88 2024-11-13T16:11:25,028 INFO [StoreOpener-29df624bbcb878edce7fcdf0abceea88-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 29df624bbcb878edce7fcdf0abceea88 columnFamilyName colfamily31 2024-11-13T16:11:25,028 DEBUG [StoreOpener-29df624bbcb878edce7fcdf0abceea88-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:11:25,031 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=72, resume processing ppid=68 2024-11-13T16:11:25,031 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, ppid=68, state=SUCCESS, hasLock=false; OpenRegionProcedure 519ece296a21b5ddccc38a7c928780bb, server=f8f85b0bb1d8,38187,1731514088725 in 979 msec 2024-11-13T16:11:25,033 INFO [StoreOpener-29df624bbcb878edce7fcdf0abceea88-1 {}] regionserver.HStore(327): Store=29df624bbcb878edce7fcdf0abceea88/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:11:25,033 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1038): replaying wal for 29df624bbcb878edce7fcdf0abceea88 2024-11-13T16:11:25,034 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6be98d350311c17b11fdb9e9a1238fca/.tmp/colfamily11/b229f1a38725438ebe67fb1caf2fcc7e as hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6be98d350311c17b11fdb9e9a1238fca/colfamily11/b229f1a38725438ebe67fb1caf2fcc7e 2024-11-13T16:11:25,036 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/29df624bbcb878edce7fcdf0abceea88 2024-11-13T16:11:25,042 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=519ece296a21b5ddccc38a7c928780bb, ASSIGN in 1.1610 sec 2024-11-13T16:11:25,046 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/29df624bbcb878edce7fcdf0abceea88 2024-11-13T16:11:25,055 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/29df624bbcb878edce7fcdf0abceea88/recovered.edits/0000000000000000004 2024-11-13T16:11:25,076 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/29df624bbcb878edce7fcdf0abceea88/recovered.edits/0000000000000000004 2024-11-13T16:11:25,077 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(2902): Flushing 29df624bbcb878edce7fcdf0abceea88 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T16:11:25,088 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6be98d350311c17b11fdb9e9a1238fca/colfamily11/b229f1a38725438ebe67fb1caf2fcc7e, entries=703, sequenceid=4, filesize=38.3 K 2024-11-13T16:11:25,088 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 6be98d350311c17b11fdb9e9a1238fca in 265ms, sequenceid=4, compaction requested=false; wal=null 2024-11-13T16:11:25,091 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6be98d350311c17b11fdb9e9a1238fca/recovered.edits/0000000000000000004 2024-11-13T16:11:25,111 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for 6be98d350311c17b11fdb9e9a1238fca 2024-11-13T16:11:25,111 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for 6be98d350311c17b11fdb9e9a1238fca 2024-11-13T16:11:25,119 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T16:11:25,141 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for 6be98d350311c17b11fdb9e9a1238fca 2024-11-13T16:11:25,156 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/29df624bbcb878edce7fcdf0abceea88/.tmp/colfamily11/0e747640257b494b954e78193607cc27 is 52, key is ooo/colfamily11:colfamily11/1731514098410/Put/seqid=0 2024-11-13T16:11:25,183 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6be98d350311c17b11fdb9e9a1238fca/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T16:11:25,207 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened 6be98d350311c17b11fdb9e9a1238fca; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60266730, jitterRate=-0.10195574164390564}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T16:11:25,207 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6be98d350311c17b11fdb9e9a1238fca 2024-11-13T16:11:25,207 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for 6be98d350311c17b11fdb9e9a1238fca: Running coprocessor pre-open hook at 1731514284706Writing region info on filesystem at 1731514284706Initializing all the Stores at 1731514284713 (+7 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514284713Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514284713Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514284713Obtaining lock to block concurrent updates at 1731514284823 (+110 ms)Preparing flush snapshotting stores in 6be98d350311c17b11fdb9e9a1238fca at 1731514284823Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731514091776.6be98d350311c17b11fdb9e9a1238fca., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731514284823Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731514091776.6be98d350311c17b11fdb9e9a1238fca. at 1731514284823Flushing 6be98d350311c17b11fdb9e9a1238fca/colfamily11: creating writer at 1731514284823Flushing 6be98d350311c17b11fdb9e9a1238fca/colfamily11: appending metadata at 1731514284879 (+56 ms)Flushing 6be98d350311c17b11fdb9e9a1238fca/colfamily11: closing flushed file at 1731514284879Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@fcf976e: reopening flushed file at 1731514285020 (+141 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 6be98d350311c17b11fdb9e9a1238fca in 265ms, sequenceid=4, compaction requested=false; wal=null at 1731514285088 (+68 ms)Cleaning up temporary data from old regions at 1731514285111 (+23 ms)Running coprocessor post-open hooks at 1731514285207 (+96 ms)Region opened successfully at 1731514285207 2024-11-13T16:11:25,213 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731514091776.6be98d350311c17b11fdb9e9a1238fca., pid=79, masterSystemTime=1731514284282 2024-11-13T16:11:25,217 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731514091776.6be98d350311c17b11fdb9e9a1238fca. 2024-11-13T16:11:25,217 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731514091776.6be98d350311c17b11fdb9e9a1238fca. 2024-11-13T16:11:25,217 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731514091776.fec5f21b48c36bf01685ec2ee58f0490. 2024-11-13T16:11:25,218 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => fec5f21b48c36bf01685ec2ee58f0490, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731514091776.fec5f21b48c36bf01685ec2ee58f0490.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-13T16:11:25,218 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true fec5f21b48c36bf01685ec2ee58f0490 2024-11-13T16:11:25,218 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731514091776.fec5f21b48c36bf01685ec2ee58f0490.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:11:25,218 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for fec5f21b48c36bf01685ec2ee58f0490 2024-11-13T16:11:25,219 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for fec5f21b48c36bf01685ec2ee58f0490 2024-11-13T16:11:25,219 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=6be98d350311c17b11fdb9e9a1238fca, regionState=OPEN, openSeqNum=5, regionLocation=f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:11:25,227 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=79, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6be98d350311c17b11fdb9e9a1238fca, server=f8f85b0bb1d8,41333,1731514088798 because future has completed 2024-11-13T16:11:25,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741881_1057 (size=39263) 2024-11-13T16:11:25,237 INFO [StoreOpener-fec5f21b48c36bf01685ec2ee58f0490-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region fec5f21b48c36bf01685ec2ee58f0490 2024-11-13T16:11:25,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741881_1057 (size=39263) 2024-11-13T16:11:25,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741881_1057 (size=39263) 2024-11-13T16:11:25,245 INFO [StoreOpener-fec5f21b48c36bf01685ec2ee58f0490-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fec5f21b48c36bf01685ec2ee58f0490 columnFamilyName colfamily11 2024-11-13T16:11:25,245 DEBUG [StoreOpener-fec5f21b48c36bf01685ec2ee58f0490-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:11:25,247 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/29df624bbcb878edce7fcdf0abceea88/.tmp/colfamily11/0e747640257b494b954e78193607cc27 2024-11-13T16:11:25,251 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=69 2024-11-13T16:11:25,251 INFO [StoreOpener-fec5f21b48c36bf01685ec2ee58f0490-1 {}] regionserver.HStore(327): Store=fec5f21b48c36bf01685ec2ee58f0490/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:11:25,252 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=69, state=SUCCESS, hasLock=false; OpenRegionProcedure 6be98d350311c17b11fdb9e9a1238fca, server=f8f85b0bb1d8,41333,1731514088798 in 1.0890 sec 2024-11-13T16:11:25,252 INFO [StoreOpener-fec5f21b48c36bf01685ec2ee58f0490-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region fec5f21b48c36bf01685ec2ee58f0490 2024-11-13T16:11:25,258 INFO [StoreOpener-fec5f21b48c36bf01685ec2ee58f0490-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fec5f21b48c36bf01685ec2ee58f0490 columnFamilyName colfamily21 2024-11-13T16:11:25,259 DEBUG [StoreOpener-fec5f21b48c36bf01685ec2ee58f0490-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:11:25,263 INFO [StoreOpener-fec5f21b48c36bf01685ec2ee58f0490-1 {}] regionserver.HStore(327): Store=fec5f21b48c36bf01685ec2ee58f0490/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:11:25,263 INFO [StoreOpener-fec5f21b48c36bf01685ec2ee58f0490-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region fec5f21b48c36bf01685ec2ee58f0490 2024-11-13T16:11:25,266 INFO [StoreOpener-fec5f21b48c36bf01685ec2ee58f0490-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fec5f21b48c36bf01685ec2ee58f0490 columnFamilyName colfamily31 2024-11-13T16:11:25,266 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=6be98d350311c17b11fdb9e9a1238fca, ASSIGN in 1.3810 sec 2024-11-13T16:11:25,266 DEBUG [StoreOpener-fec5f21b48c36bf01685ec2ee58f0490-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:11:25,268 INFO [StoreOpener-fec5f21b48c36bf01685ec2ee58f0490-1 {}] regionserver.HStore(327): Store=fec5f21b48c36bf01685ec2ee58f0490/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:11:25,268 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for fec5f21b48c36bf01685ec2ee58f0490 2024-11-13T16:11:25,271 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fec5f21b48c36bf01685ec2ee58f0490 2024-11-13T16:11:25,296 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fec5f21b48c36bf01685ec2ee58f0490 2024-11-13T16:11:25,297 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/29df624bbcb878edce7fcdf0abceea88/.tmp/colfamily11/0e747640257b494b954e78193607cc27 as hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/29df624bbcb878edce7fcdf0abceea88/colfamily11/0e747640257b494b954e78193607cc27 2024-11-13T16:11:25,304 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fec5f21b48c36bf01685ec2ee58f0490/recovered.edits/0000000000000000004 2024-11-13T16:11:25,312 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/29df624bbcb878edce7fcdf0abceea88/colfamily11/0e747640257b494b954e78193607cc27, entries=703, sequenceid=4, filesize=38.3 K 2024-11-13T16:11:25,312 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 29df624bbcb878edce7fcdf0abceea88 in 235ms, sequenceid=4, compaction requested=false; wal=null 2024-11-13T16:11:25,315 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/29df624bbcb878edce7fcdf0abceea88/recovered.edits/0000000000000000004 2024-11-13T16:11:25,319 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1048): stopping wal replay for 29df624bbcb878edce7fcdf0abceea88 2024-11-13T16:11:25,320 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1060): Cleaning up temporary data for 29df624bbcb878edce7fcdf0abceea88 2024-11-13T16:11:25,325 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fec5f21b48c36bf01685ec2ee58f0490/recovered.edits/0000000000000000004 2024-11-13T16:11:25,325 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(2902): Flushing fec5f21b48c36bf01685ec2ee58f0490 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T16:11:25,325 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T16:11:25,339 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1093): writing seq id for 29df624bbcb878edce7fcdf0abceea88 2024-11-13T16:11:25,360 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/29df624bbcb878edce7fcdf0abceea88/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T16:11:25,366 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1114): Opened 29df624bbcb878edce7fcdf0abceea88; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72353409, jitterRate=0.07814981043338776}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T16:11:25,366 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 29df624bbcb878edce7fcdf0abceea88 2024-11-13T16:11:25,366 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1006): Region open journal for 29df624bbcb878edce7fcdf0abceea88: Running coprocessor pre-open hook at 1731514284993Writing region info on filesystem at 1731514284993Initializing all the Stores at 1731514284998 (+5 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514284998Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514284998Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514284998Obtaining lock to block concurrent updates at 1731514285077 (+79 ms)Preparing flush snapshotting stores in 29df624bbcb878edce7fcdf0abceea88 at 1731514285077Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731514091776.29df624bbcb878edce7fcdf0abceea88., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731514285077Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731514091776.29df624bbcb878edce7fcdf0abceea88. at 1731514285077Flushing 29df624bbcb878edce7fcdf0abceea88/colfamily11: creating writer at 1731514285077Flushing 29df624bbcb878edce7fcdf0abceea88/colfamily11: appending metadata at 1731514285154 (+77 ms)Flushing 29df624bbcb878edce7fcdf0abceea88/colfamily11: closing flushed file at 1731514285154Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b37fba3: reopening flushed file at 1731514285275 (+121 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 29df624bbcb878edce7fcdf0abceea88 in 235ms, sequenceid=4, compaction requested=false; wal=null at 1731514285312 (+37 ms)Cleaning up temporary data from old regions at 1731514285320 (+8 ms)Running coprocessor post-open hooks at 1731514285366 (+46 ms)Region opened successfully at 1731514285366 2024-11-13T16:11:25,371 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731514091776.29df624bbcb878edce7fcdf0abceea88., pid=74, masterSystemTime=1731514284196 2024-11-13T16:11:25,391 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731514091776.29df624bbcb878edce7fcdf0abceea88. 2024-11-13T16:11:25,391 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731514091776.29df624bbcb878edce7fcdf0abceea88. 2024-11-13T16:11:25,391 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731514091776.e8725a794e6d251e1bf6e3502baa7700. 2024-11-13T16:11:25,391 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7752): Opening region: {ENCODED => e8725a794e6d251e1bf6e3502baa7700, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731514091776.e8725a794e6d251e1bf6e3502baa7700.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-13T16:11:25,392 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true e8725a794e6d251e1bf6e3502baa7700 2024-11-13T16:11:25,392 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731514091776.e8725a794e6d251e1bf6e3502baa7700.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:11:25,392 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7794): checking encryption for e8725a794e6d251e1bf6e3502baa7700 2024-11-13T16:11:25,392 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7797): checking classloading for e8725a794e6d251e1bf6e3502baa7700 2024-11-13T16:11:25,393 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=29df624bbcb878edce7fcdf0abceea88, regionState=OPEN, openSeqNum=5, regionLocation=f8f85b0bb1d8,38187,1731514088725 2024-11-13T16:11:25,401 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fec5f21b48c36bf01685ec2ee58f0490/.tmp/colfamily11/5d7d06c1dfc94aa3b53fa2f6a13c155e is 52, key is sss/colfamily11:colfamily11/1731514098449/Put/seqid=0 2024-11-13T16:11:25,402 INFO [StoreOpener-e8725a794e6d251e1bf6e3502baa7700-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region e8725a794e6d251e1bf6e3502baa7700 2024-11-13T16:11:25,403 INFO [StoreOpener-e8725a794e6d251e1bf6e3502baa7700-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e8725a794e6d251e1bf6e3502baa7700 columnFamilyName colfamily11 2024-11-13T16:11:25,404 DEBUG [StoreOpener-e8725a794e6d251e1bf6e3502baa7700-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:11:25,404 INFO [StoreOpener-e8725a794e6d251e1bf6e3502baa7700-1 {}] regionserver.HStore(327): Store=e8725a794e6d251e1bf6e3502baa7700/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:11:25,405 INFO [StoreOpener-e8725a794e6d251e1bf6e3502baa7700-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region e8725a794e6d251e1bf6e3502baa7700 2024-11-13T16:11:25,406 INFO [StoreOpener-e8725a794e6d251e1bf6e3502baa7700-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e8725a794e6d251e1bf6e3502baa7700 columnFamilyName colfamily21 2024-11-13T16:11:25,406 DEBUG [StoreOpener-e8725a794e6d251e1bf6e3502baa7700-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:11:25,407 INFO [StoreOpener-e8725a794e6d251e1bf6e3502baa7700-1 {}] regionserver.HStore(327): Store=e8725a794e6d251e1bf6e3502baa7700/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:11:25,407 INFO [StoreOpener-e8725a794e6d251e1bf6e3502baa7700-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region e8725a794e6d251e1bf6e3502baa7700 2024-11-13T16:11:25,409 INFO [StoreOpener-e8725a794e6d251e1bf6e3502baa7700-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e8725a794e6d251e1bf6e3502baa7700 columnFamilyName colfamily31 2024-11-13T16:11:25,410 DEBUG [StoreOpener-e8725a794e6d251e1bf6e3502baa7700-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:11:25,411 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=74, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 29df624bbcb878edce7fcdf0abceea88, server=f8f85b0bb1d8,38187,1731514088725 because future has completed 2024-11-13T16:11:25,413 INFO [StoreOpener-e8725a794e6d251e1bf6e3502baa7700-1 {}] regionserver.HStore(327): Store=e8725a794e6d251e1bf6e3502baa7700/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:11:25,415 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1038): replaying wal for e8725a794e6d251e1bf6e3502baa7700 2024-11-13T16:11:25,418 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e8725a794e6d251e1bf6e3502baa7700 2024-11-13T16:11:25,428 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e8725a794e6d251e1bf6e3502baa7700 2024-11-13T16:11:25,432 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e8725a794e6d251e1bf6e3502baa7700/recovered.edits/0000000000000000004 2024-11-13T16:11:25,441 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e8725a794e6d251e1bf6e3502baa7700/recovered.edits/0000000000000000004 2024-11-13T16:11:25,441 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(2902): Flushing e8725a794e6d251e1bf6e3502baa7700 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T16:11:25,443 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=74, resume processing ppid=70 2024-11-13T16:11:25,443 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, ppid=70, state=SUCCESS, hasLock=false; OpenRegionProcedure 29df624bbcb878edce7fcdf0abceea88, server=f8f85b0bb1d8,38187,1731514088725 in 1.3650 sec 2024-11-13T16:11:25,445 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=29df624bbcb878edce7fcdf0abceea88, ASSIGN in 1.5720 sec 2024-11-13T16:11:25,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741882_1058 (size=39263) 2024-11-13T16:11:25,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741882_1058 (size=39263) 2024-11-13T16:11:25,476 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fec5f21b48c36bf01685ec2ee58f0490/.tmp/colfamily11/5d7d06c1dfc94aa3b53fa2f6a13c155e 2024-11-13T16:11:25,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741882_1058 (size=39263) 2024-11-13T16:11:25,491 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e8725a794e6d251e1bf6e3502baa7700/.tmp/colfamily11/178a46164d51486eb98c9a1926df3e7c is 52, key is iii/colfamily11:colfamily11/1731514098331/Put/seqid=0 2024-11-13T16:11:25,513 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fec5f21b48c36bf01685ec2ee58f0490/.tmp/colfamily11/5d7d06c1dfc94aa3b53fa2f6a13c155e as hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fec5f21b48c36bf01685ec2ee58f0490/colfamily11/5d7d06c1dfc94aa3b53fa2f6a13c155e 2024-11-13T16:11:25,534 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fec5f21b48c36bf01685ec2ee58f0490/colfamily11/5d7d06c1dfc94aa3b53fa2f6a13c155e, entries=703, sequenceid=4, filesize=38.3 K 2024-11-13T16:11:25,535 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for fec5f21b48c36bf01685ec2ee58f0490 in 209ms, sequenceid=4, compaction requested=false; wal=null 2024-11-13T16:11:25,539 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fec5f21b48c36bf01685ec2ee58f0490/recovered.edits/0000000000000000004 2024-11-13T16:11:25,547 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for fec5f21b48c36bf01685ec2ee58f0490 2024-11-13T16:11:25,547 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for fec5f21b48c36bf01685ec2ee58f0490 2024-11-13T16:11:25,549 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T16:11:25,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741883_1059 (size=39263) 2024-11-13T16:11:25,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741883_1059 (size=39263) 2024-11-13T16:11:25,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741883_1059 (size=39263) 2024-11-13T16:11:25,554 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e8725a794e6d251e1bf6e3502baa7700/.tmp/colfamily11/178a46164d51486eb98c9a1926df3e7c 2024-11-13T16:11:25,556 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for fec5f21b48c36bf01685ec2ee58f0490 2024-11-13T16:11:25,560 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fec5f21b48c36bf01685ec2ee58f0490/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T16:11:25,562 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened fec5f21b48c36bf01685ec2ee58f0490; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74604878, jitterRate=0.11169931292533875}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T16:11:25,563 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fec5f21b48c36bf01685ec2ee58f0490 2024-11-13T16:11:25,563 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for fec5f21b48c36bf01685ec2ee58f0490: Running coprocessor pre-open hook at 1731514285219Writing region info on filesystem at 1731514285219Initializing all the Stores at 1731514285230 (+11 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514285231 (+1 ms)Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514285235 (+4 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514285235Obtaining lock to block concurrent updates at 1731514285325 (+90 ms)Preparing flush snapshotting stores in fec5f21b48c36bf01685ec2ee58f0490 at 1731514285325Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731514091776.fec5f21b48c36bf01685ec2ee58f0490., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731514285325Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731514091776.fec5f21b48c36bf01685ec2ee58f0490. at 1731514285325Flushing fec5f21b48c36bf01685ec2ee58f0490/colfamily11: creating writer at 1731514285325Flushing fec5f21b48c36bf01685ec2ee58f0490/colfamily11: appending metadata at 1731514285400 (+75 ms)Flushing fec5f21b48c36bf01685ec2ee58f0490/colfamily11: closing flushed file at 1731514285400Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@50a71ebb: reopening flushed file at 1731514285509 (+109 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for fec5f21b48c36bf01685ec2ee58f0490 in 209ms, sequenceid=4, compaction requested=false; wal=null at 1731514285535 (+26 ms)Cleaning up temporary data from old regions at 1731514285547 (+12 ms)Running coprocessor post-open hooks at 1731514285563 (+16 ms)Region opened successfully at 1731514285563 2024-11-13T16:11:25,565 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e8725a794e6d251e1bf6e3502baa7700/.tmp/colfamily11/178a46164d51486eb98c9a1926df3e7c as hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e8725a794e6d251e1bf6e3502baa7700/colfamily11/178a46164d51486eb98c9a1926df3e7c 2024-11-13T16:11:25,568 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731514091776.fec5f21b48c36bf01685ec2ee58f0490., pid=78, masterSystemTime=1731514284282 2024-11-13T16:11:25,574 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731514091776.fec5f21b48c36bf01685ec2ee58f0490. 2024-11-13T16:11:25,574 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731514091776.fec5f21b48c36bf01685ec2ee58f0490. 2024-11-13T16:11:25,574 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=fec5f21b48c36bf01685ec2ee58f0490, regionState=OPEN, openSeqNum=5, regionLocation=f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:11:25,574 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731514091776.aa124b802c708ca1f715cb1f8c9c4eaa. 2024-11-13T16:11:25,574 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => aa124b802c708ca1f715cb1f8c9c4eaa, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731514091776.aa124b802c708ca1f715cb1f8c9c4eaa.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-13T16:11:25,575 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true aa124b802c708ca1f715cb1f8c9c4eaa 2024-11-13T16:11:25,575 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731514091776.aa124b802c708ca1f715cb1f8c9c4eaa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:11:25,575 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for aa124b802c708ca1f715cb1f8c9c4eaa 2024-11-13T16:11:25,575 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for aa124b802c708ca1f715cb1f8c9c4eaa 2024-11-13T16:11:25,576 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e8725a794e6d251e1bf6e3502baa7700/colfamily11/178a46164d51486eb98c9a1926df3e7c, entries=703, sequenceid=4, filesize=38.3 K 2024-11-13T16:11:25,576 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for e8725a794e6d251e1bf6e3502baa7700 in 135ms, sequenceid=4, compaction requested=false; wal=null 2024-11-13T16:11:25,577 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure fec5f21b48c36bf01685ec2ee58f0490, server=f8f85b0bb1d8,41333,1731514088798 because future has completed 2024-11-13T16:11:25,577 INFO [StoreOpener-aa124b802c708ca1f715cb1f8c9c4eaa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region aa124b802c708ca1f715cb1f8c9c4eaa 2024-11-13T16:11:25,578 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e8725a794e6d251e1bf6e3502baa7700/recovered.edits/0000000000000000004 2024-11-13T16:11:25,579 INFO [StoreOpener-aa124b802c708ca1f715cb1f8c9c4eaa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aa124b802c708ca1f715cb1f8c9c4eaa columnFamilyName colfamily11 2024-11-13T16:11:25,579 DEBUG [StoreOpener-aa124b802c708ca1f715cb1f8c9c4eaa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:11:25,580 INFO [StoreOpener-aa124b802c708ca1f715cb1f8c9c4eaa-1 {}] regionserver.HStore(327): Store=aa124b802c708ca1f715cb1f8c9c4eaa/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:11:25,580 INFO [StoreOpener-aa124b802c708ca1f715cb1f8c9c4eaa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region aa124b802c708ca1f715cb1f8c9c4eaa 2024-11-13T16:11:25,580 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1048): stopping wal replay for e8725a794e6d251e1bf6e3502baa7700 2024-11-13T16:11:25,580 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1060): Cleaning up temporary data for e8725a794e6d251e1bf6e3502baa7700 2024-11-13T16:11:25,581 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T16:11:25,581 INFO [StoreOpener-aa124b802c708ca1f715cb1f8c9c4eaa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aa124b802c708ca1f715cb1f8c9c4eaa columnFamilyName colfamily21 2024-11-13T16:11:25,581 DEBUG [StoreOpener-aa124b802c708ca1f715cb1f8c9c4eaa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:11:25,582 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42949 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=f8f85b0bb1d8,41333,1731514088798, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fec5f21b48c36bf01685ec2ee58f0490. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T16:11:25,582 INFO [StoreOpener-aa124b802c708ca1f715cb1f8c9c4eaa-1 {}] regionserver.HStore(327): Store=aa124b802c708ca1f715cb1f8c9c4eaa/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:11:25,583 INFO [StoreOpener-aa124b802c708ca1f715cb1f8c9c4eaa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region aa124b802c708ca1f715cb1f8c9c4eaa 2024-11-13T16:11:25,585 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1093): writing seq id for e8725a794e6d251e1bf6e3502baa7700 2024-11-13T16:11:25,585 INFO [StoreOpener-aa124b802c708ca1f715cb1f8c9c4eaa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aa124b802c708ca1f715cb1f8c9c4eaa columnFamilyName colfamily31 2024-11-13T16:11:25,585 DEBUG [StoreOpener-aa124b802c708ca1f715cb1f8c9c4eaa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:11:25,586 INFO [StoreOpener-aa124b802c708ca1f715cb1f8c9c4eaa-1 {}] regionserver.HStore(327): Store=aa124b802c708ca1f715cb1f8c9c4eaa/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:11:25,586 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for aa124b802c708ca1f715cb1f8c9c4eaa 2024-11-13T16:11:25,589 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aa124b802c708ca1f715cb1f8c9c4eaa 2024-11-13T16:11:25,591 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e8725a794e6d251e1bf6e3502baa7700/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T16:11:25,593 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1114): Opened e8725a794e6d251e1bf6e3502baa7700; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58921782, jitterRate=-0.12199702858924866}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T16:11:25,594 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e8725a794e6d251e1bf6e3502baa7700 2024-11-13T16:11:25,594 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1006): Region open journal for e8725a794e6d251e1bf6e3502baa7700: Running coprocessor pre-open hook at 1731514285392Writing region info on filesystem at 1731514285392Initializing all the Stores at 1731514285397 (+5 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514285398 (+1 ms)Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514285402 (+4 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514285402Obtaining lock to block concurrent updates at 1731514285441 (+39 ms)Preparing flush snapshotting stores in e8725a794e6d251e1bf6e3502baa7700 at 1731514285441Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731514091776.e8725a794e6d251e1bf6e3502baa7700., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731514285442 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731514091776.e8725a794e6d251e1bf6e3502baa7700. at 1731514285442Flushing e8725a794e6d251e1bf6e3502baa7700/colfamily11: creating writer at 1731514285442Flushing e8725a794e6d251e1bf6e3502baa7700/colfamily11: appending metadata at 1731514285490 (+48 ms)Flushing e8725a794e6d251e1bf6e3502baa7700/colfamily11: closing flushed file at 1731514285490Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@633c7956: reopening flushed file at 1731514285564 (+74 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for e8725a794e6d251e1bf6e3502baa7700 in 135ms, sequenceid=4, compaction requested=false; wal=null at 1731514285577 (+13 ms)Cleaning up temporary data from old regions at 1731514285580 (+3 ms)Running coprocessor post-open hooks at 1731514285594 (+14 ms)Region opened successfully at 1731514285594 2024-11-13T16:11:25,603 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=78, resume processing ppid=71 2024-11-13T16:11:25,603 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, ppid=71, state=SUCCESS, hasLock=false; OpenRegionProcedure fec5f21b48c36bf01685ec2ee58f0490, server=f8f85b0bb1d8,41333,1731514088798 in 1.4430 sec 2024-11-13T16:11:25,604 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731514091776.e8725a794e6d251e1bf6e3502baa7700., pid=73, masterSystemTime=1731514284196 2024-11-13T16:11:25,604 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aa124b802c708ca1f715cb1f8c9c4eaa 2024-11-13T16:11:25,608 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aa124b802c708ca1f715cb1f8c9c4eaa/recovered.edits/0000000000000000004 2024-11-13T16:11:25,609 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fec5f21b48c36bf01685ec2ee58f0490, ASSIGN in 1.7330 sec 2024-11-13T16:11:25,609 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731514091776.e8725a794e6d251e1bf6e3502baa7700. 2024-11-13T16:11:25,610 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731514091776.e8725a794e6d251e1bf6e3502baa7700. 2024-11-13T16:11:25,611 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=e8725a794e6d251e1bf6e3502baa7700, regionState=OPEN, openSeqNum=5, regionLocation=f8f85b0bb1d8,38187,1731514088725 2024-11-13T16:11:25,620 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aa124b802c708ca1f715cb1f8c9c4eaa/recovered.edits/0000000000000000004 2024-11-13T16:11:25,620 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(2902): Flushing aa124b802c708ca1f715cb1f8c9c4eaa 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T16:11:25,622 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure e8725a794e6d251e1bf6e3502baa7700, server=f8f85b0bb1d8,38187,1731514088725 because future has completed 2024-11-13T16:11:25,649 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aa124b802c708ca1f715cb1f8c9c4eaa/.tmp/colfamily11/7531c0cc27534e8bb7847988a8e7f51b is 52, key is mmm/colfamily11:colfamily11/1731514098375/Put/seqid=0 2024-11-13T16:11:25,655 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=66 2024-11-13T16:11:25,655 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=66, state=SUCCESS, hasLock=false; OpenRegionProcedure e8725a794e6d251e1bf6e3502baa7700, server=f8f85b0bb1d8,38187,1731514088725 in 1.5840 sec 2024-11-13T16:11:25,658 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e8725a794e6d251e1bf6e3502baa7700, ASSIGN in 1.7840 sec 2024-11-13T16:11:25,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741884_1060 (size=39263) 2024-11-13T16:11:25,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741884_1060 (size=39263) 2024-11-13T16:11:25,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741884_1060 (size=39263) 2024-11-13T16:11:26,070 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T16:11:26,110 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aa124b802c708ca1f715cb1f8c9c4eaa/.tmp/colfamily11/7531c0cc27534e8bb7847988a8e7f51b 2024-11-13T16:11:26,128 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aa124b802c708ca1f715cb1f8c9c4eaa/.tmp/colfamily11/7531c0cc27534e8bb7847988a8e7f51b as hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aa124b802c708ca1f715cb1f8c9c4eaa/colfamily11/7531c0cc27534e8bb7847988a8e7f51b 2024-11-13T16:11:26,139 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aa124b802c708ca1f715cb1f8c9c4eaa/colfamily11/7531c0cc27534e8bb7847988a8e7f51b, entries=703, sequenceid=4, filesize=38.3 K 2024-11-13T16:11:26,139 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for aa124b802c708ca1f715cb1f8c9c4eaa in 519ms, sequenceid=4, compaction requested=false; wal=null 2024-11-13T16:11:26,140 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aa124b802c708ca1f715cb1f8c9c4eaa/recovered.edits/0000000000000000004 2024-11-13T16:11:26,152 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for aa124b802c708ca1f715cb1f8c9c4eaa 2024-11-13T16:11:26,152 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for aa124b802c708ca1f715cb1f8c9c4eaa 2024-11-13T16:11:26,163 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T16:11:26,174 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for aa124b802c708ca1f715cb1f8c9c4eaa 2024-11-13T16:11:26,183 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aa124b802c708ca1f715cb1f8c9c4eaa/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T16:11:26,186 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened aa124b802c708ca1f715cb1f8c9c4eaa; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66173661, jitterRate=-0.01393561065196991}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T16:11:26,186 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for aa124b802c708ca1f715cb1f8c9c4eaa 2024-11-13T16:11:26,186 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for aa124b802c708ca1f715cb1f8c9c4eaa: Running coprocessor pre-open hook at 1731514285575Writing region info on filesystem at 1731514285575Initializing all the Stores at 1731514285577 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514285577Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514285577Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514285577Obtaining lock to block concurrent updates at 1731514285620 (+43 ms)Preparing flush snapshotting stores in aa124b802c708ca1f715cb1f8c9c4eaa at 1731514285620Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731514091776.aa124b802c708ca1f715cb1f8c9c4eaa., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731514285621 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731514091776.aa124b802c708ca1f715cb1f8c9c4eaa. at 1731514285621Flushing aa124b802c708ca1f715cb1f8c9c4eaa/colfamily11: creating writer at 1731514285621Flushing aa124b802c708ca1f715cb1f8c9c4eaa/colfamily11: appending metadata at 1731514285648 (+27 ms)Flushing aa124b802c708ca1f715cb1f8c9c4eaa/colfamily11: closing flushed file at 1731514285648Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7401a26e: reopening flushed file at 1731514286125 (+477 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for aa124b802c708ca1f715cb1f8c9c4eaa in 519ms, sequenceid=4, compaction requested=false; wal=null at 1731514286139 (+14 ms)Cleaning up temporary data from old regions at 1731514286152 (+13 ms)Running coprocessor post-open hooks at 1731514286186 (+34 ms)Region opened successfully at 1731514286186 2024-11-13T16:11:26,191 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731514091776.aa124b802c708ca1f715cb1f8c9c4eaa., pid=77, masterSystemTime=1731514284282 2024-11-13T16:11:26,193 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731514091776.aa124b802c708ca1f715cb1f8c9c4eaa. 2024-11-13T16:11:26,193 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731514091776.aa124b802c708ca1f715cb1f8c9c4eaa. 2024-11-13T16:11:26,198 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=aa124b802c708ca1f715cb1f8c9c4eaa, regionState=OPEN, openSeqNum=5, regionLocation=f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:11:26,201 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42949 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=f8f85b0bb1d8,41333,1731514088798, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=aa124b802c708ca1f715cb1f8c9c4eaa. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T16:11:26,204 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=77, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure aa124b802c708ca1f715cb1f8c9c4eaa, server=f8f85b0bb1d8,41333,1731514088798 because future has completed 2024-11-13T16:11:26,216 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=64 2024-11-13T16:11:26,216 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure aa124b802c708ca1f715cb1f8c9c4eaa, server=f8f85b0bb1d8,41333,1731514088798 in 2.0670 sec 2024-11-13T16:11:26,219 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=57 2024-11-13T16:11:26,219 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=aa124b802c708ca1f715cb1f8c9c4eaa, ASSIGN in 2.3450 sec 2024-11-13T16:11:26,219 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server f8f85b0bb1d8,43467,1731514088560 after splitting done 2024-11-13T16:11:26,219 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-13T16:11:26,219 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=SUCCESS, hasLock=true; ServerCrashProcedure f8f85b0bb1d8,43467,1731514088560, splitWal=true, meta=true 2024-11-13T16:11:26,219 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T16:11:26,220 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=SUCCESS, hasLock=true; ServerCrashProcedure f8f85b0bb1d8,43467,1731514088560, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T16:11:26,385 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 8 exec state=pid=57, state=SUCCESS, hasLock=true; ServerCrashProcedure f8f85b0bb1d8,43467,1731514088560, splitWal=true, meta=true 2024-11-13T16:11:26,385 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-13T16:11:26,385 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T16:11:26,385 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T16:11:26,385 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T16:11:26,386 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-13T16:11:26,386 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-13T16:11:26,386 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-13T16:11:26,386 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T16:11:26,387 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-13T16:11:26,389 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-13T16:11:26,389 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T16:11:26,389 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T16:11:26,389 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T16:11:26,389 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T16:11:26,389 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T16:11:26,389 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T16:11:26,389 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T16:11:26,389 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T16:11:26,389 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T16:11:26,390 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T16:11:26,390 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T16:11:26,390 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T16:11:26,390 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T16:11:26,390 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T16:11:26,390 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T16:11:26,390 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T16:11:26,390 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T16:11:26,390 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T16:11:26,390 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T16:11:26,390 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T16:11:26,390 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T16:11:26,390 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T16:11:26,390 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T16:11:26,391 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-13T16:11:26,394 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_FINISH, hasLock=false; ServerCrashProcedure f8f85b0bb1d8,43467,1731514088560, splitWal=true, meta=true 2024-11-13T16:11:26,394 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure f8f85b0bb1d8%2C43467%2C1731514088560.meta.1731514091182.meta, worker=f8f85b0bb1d8,38187,1731514088725 2024-11-13T16:11:26,394 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f8f85b0bb1d8%2C43467%2C1731514088560.meta.1731514091182.meta, worker=f8f85b0bb1d8,38187,1731514088725 2024-11-13T16:11:26,395 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T16:11:26,395 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:11:26,395 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856, worker=f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:11:26,395 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856, worker=f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:11:26,395 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=aa124b802c708ca1f715cb1f8c9c4eaa, ASSIGN 2024-11-13T16:11:26,395 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8ba796d80fc3b65f0948c95ccbdad39d, ASSIGN 2024-11-13T16:11:26,395 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e8725a794e6d251e1bf6e3502baa7700, ASSIGN 2024-11-13T16:11:26,395 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3945ce860c8642051e17517944a83136, ASSIGN 2024-11-13T16:11:26,395 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=519ece296a21b5ddccc38a7c928780bb, ASSIGN 2024-11-13T16:11:26,395 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=6be98d350311c17b11fdb9e9a1238fca, ASSIGN 2024-11-13T16:11:26,395 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=29df624bbcb878edce7fcdf0abceea88, ASSIGN 2024-11-13T16:11:26,395 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fec5f21b48c36bf01685ec2ee58f0490, ASSIGN 2024-11-13T16:11:26,395 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=68, state=SUCCESS, hasLock=false; OpenRegionProcedure 519ece296a21b5ddccc38a7c928780bb, server=f8f85b0bb1d8,38187,1731514088725 2024-11-13T16:11:26,395 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=66, state=SUCCESS, hasLock=false; OpenRegionProcedure e8725a794e6d251e1bf6e3502baa7700, server=f8f85b0bb1d8,38187,1731514088725 2024-11-13T16:11:26,395 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=70, state=SUCCESS, hasLock=false; OpenRegionProcedure 29df624bbcb878edce7fcdf0abceea88, server=f8f85b0bb1d8,38187,1731514088725 2024-11-13T16:11:26,395 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=65, state=SUCCESS, hasLock=false; OpenRegionProcedure 8ba796d80fc3b65f0948c95ccbdad39d, server=f8f85b0bb1d8,38187,1731514088725 2024-11-13T16:11:26,395 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=67, state=SUCCESS, hasLock=false; OpenRegionProcedure 3945ce860c8642051e17517944a83136, server=f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:11:26,395 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure aa124b802c708ca1f715cb1f8c9c4eaa, server=f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:11:26,395 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=71, state=SUCCESS, hasLock=false; OpenRegionProcedure fec5f21b48c36bf01685ec2ee58f0490, server=f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:11:26,395 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=69, state=SUCCESS, hasLock=false; OpenRegionProcedure 6be98d350311c17b11fdb9e9a1238fca, server=f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:11:26,396 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_FINISH, hasLock=false; ServerCrashProcedure f8f85b0bb1d8,43467,1731514088560, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-13T16:11:26,396 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure f8f85b0bb1d8%2C43467%2C1731514088560.meta.1731514091182.meta, worker=f8f85b0bb1d8,38187,1731514088725 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T16:11:26,396 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f8f85b0bb1d8%2C43467%2C1731514088560.meta.1731514091182.meta, worker=f8f85b0bb1d8,38187,1731514088725 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T16:11:26,396 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T16:11:26,396 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f8f85b0bb1d8,41333,1731514088798 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T16:11:26,396 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856, worker=f8f85b0bb1d8,41333,1731514088798 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T16:11:26,396 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f8f85b0bb1d8%2C43467%2C1731514088560.1731514090856, worker=f8f85b0bb1d8,41333,1731514088798 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T16:11:26,396 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=aa124b802c708ca1f715cb1f8c9c4eaa, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T16:11:26,396 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8ba796d80fc3b65f0948c95ccbdad39d, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T16:11:26,396 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e8725a794e6d251e1bf6e3502baa7700, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T16:11:26,396 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=67, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3945ce860c8642051e17517944a83136, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T16:11:26,396 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=68, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=519ece296a21b5ddccc38a7c928780bb, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T16:11:26,396 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=69, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=6be98d350311c17b11fdb9e9a1238fca, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T16:11:26,396 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=70, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=29df624bbcb878edce7fcdf0abceea88, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T16:11:26,396 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=71, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fec5f21b48c36bf01685ec2ee58f0490, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T16:11:26,396 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=68, state=SUCCESS, hasLock=false; OpenRegionProcedure 519ece296a21b5ddccc38a7c928780bb, server=f8f85b0bb1d8,38187,1731514088725 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T16:11:26,396 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=66, state=SUCCESS, hasLock=false; OpenRegionProcedure e8725a794e6d251e1bf6e3502baa7700, server=f8f85b0bb1d8,38187,1731514088725 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T16:11:26,396 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=70, state=SUCCESS, hasLock=false; OpenRegionProcedure 29df624bbcb878edce7fcdf0abceea88, server=f8f85b0bb1d8,38187,1731514088725 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T16:11:26,396 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=65, state=SUCCESS, hasLock=false; OpenRegionProcedure 8ba796d80fc3b65f0948c95ccbdad39d, server=f8f85b0bb1d8,38187,1731514088725 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T16:11:26,396 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=67, state=SUCCESS, hasLock=false; OpenRegionProcedure 3945ce860c8642051e17517944a83136, server=f8f85b0bb1d8,41333,1731514088798 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T16:11:26,396 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure aa124b802c708ca1f715cb1f8c9c4eaa, server=f8f85b0bb1d8,41333,1731514088798 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T16:11:26,396 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=71, state=SUCCESS, hasLock=false; OpenRegionProcedure fec5f21b48c36bf01685ec2ee58f0490, server=f8f85b0bb1d8,41333,1731514088798 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T16:11:26,397 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=69, state=SUCCESS, hasLock=false; OpenRegionProcedure 6be98d350311c17b11fdb9e9a1238fca, server=f8f85b0bb1d8,41333,1731514088798 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T16:11:26,397 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-13T16:11:26,398 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=61 2024-11-13T16:11:26,398 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=f8f85b0bb1d8,41333,1731514088798, table=hbase:meta, region=1588230740 2024-11-13T16:11:26,399 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f8f85b0bb1d8,41333,1731514088798, state=OPEN 2024-11-13T16:11:26,409 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42949-0x1013621fa650000, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T16:11:26,409 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38187-0x1013621fa650002, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T16:11:26,409 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41333-0x1013621fa650003, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T16:11:26,409 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:11:26,409 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:11:26,409 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:11:26,411 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42949 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T16:11:26,413 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731514316413 2024-11-13T16:11:26,413 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-13T16:11:26,425 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4aaf5667f132c289386d1b3b0c0ac57f, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=2 2024-11-13T16:11:26,425 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ba796d80fc3b65f0948c95ccbdad39d, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=5 2024-11-13T16:11:26,426 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93f8ed38b68bd013eb0cf0e7615013ae, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=2 2024-11-13T16:11:26,426 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e72e57b96ff9bc0bbd862ce564d79c42, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=2 2024-11-13T16:11:26,426 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34f3d3b6a6dae1f4077567bed0bd8e75, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=2 2024-11-13T16:11:26,426 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3945ce860c8642051e17517944a83136, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=5 2024-11-13T16:11:26,426 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1fcf78578ccab75f1705f3c2d7bcdb14, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=2 2024-11-13T16:11:26,426 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a610c2403a0b491d368473a6e1fb049, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=2 2024-11-13T16:11:26,426 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e8725a794e6d251e1bf6e3502baa7700, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=5 2024-11-13T16:11:26,427 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa6636be3559bc84793534420cd4f207, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=2 2024-11-13T16:11:26,427 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=05d65f4a6bfecb3d530d15832c218252, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=2 2024-11-13T16:11:26,427 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78285ca993cc7fbfd78041c30558f756, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=2 2024-11-13T16:11:26,427 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa124b802c708ca1f715cb1f8c9c4eaa, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=5 2024-11-13T16:11:26,427 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3647edbd8aef23dda3887b61277da1d3, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=2 2024-11-13T16:11:26,427 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=29df624bbcb878edce7fcdf0abceea88, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=5 2024-11-13T16:11:26,428 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fe97fe0e9b37811ba99ca8100b461589, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=2 2024-11-13T16:11:26,428 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7c34e9ded13254b91545186c85befbf8, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=2 2024-11-13T16:11:26,428 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8f8b8e4ae8470fd8a1d2bf0684947ef, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=2 2024-11-13T16:11:26,428 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fec5f21b48c36bf01685ec2ee58f0490, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=5 2024-11-13T16:11:26,428 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97998e7179d85b23a3f5928cd6511629, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=2 2024-11-13T16:11:26,428 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=519ece296a21b5ddccc38a7c928780bb, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=5 2024-11-13T16:11:26,429 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4a87adf770b37d40ab6691e377ad1b5, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=2 2024-11-13T16:11:26,429 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf1f9f9bc1bc9ffce566d41f8ae3619a, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=2 2024-11-13T16:11:26,429 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a146f98682f25e8cddcae5dbf318603, regionState=OPEN, lastHost=f8f85b0bb1d8,38187,1731514088725, regionLocation=f8f85b0bb1d8,38187,1731514088725, openSeqNum=2 2024-11-13T16:11:26,429 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6be98d350311c17b11fdb9e9a1238fca, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=5 2024-11-13T16:11:26,429 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=72d273770314688e78624993de4f3398, regionState=OPEN, lastHost=f8f85b0bb1d8,41333,1731514088798, regionLocation=f8f85b0bb1d8,41333,1731514088798, openSeqNum=2 2024-11-13T16:11:26,429 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-13T16:11:26,429 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731514346429 2024-11-13T16:11:26,429 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731514406429 2024-11-13T16:11:26,430 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 16 msec 2024-11-13T16:11:26,433 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server f8f85b0bb1d8,43467,1731514088560 after splitting done 2024-11-13T16:11:26,433 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-13T16:11:26,442 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; ServerCrashProcedure f8f85b0bb1d8,43467,1731514088560, splitWal=true, meta=true in 3 mins, 6.629 sec 2024-11-13T16:11:26,691 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55790, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T16:11:26,696 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56084, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T16:11:26,702 WARN [RPCClient-NioEventLoopGroup-6-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server f8f85b0bb1d8:43467 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:11:26,704 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731514091776.8ba796d80fc3b65f0948c95ccbdad39d., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731514091776.8ba796d80fc3b65f0948c95ccbdad39d., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2, error=java.net.ConnectException: Call to address=f8f85b0bb1d8:43467 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:11:26,704 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731514091776.8ba796d80fc3b65f0948c95ccbdad39d., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2 is java.net.ConnectException: Connection refused 2024-11-13T16:11:26,704 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731514091776.8ba796d80fc3b65f0948c95ccbdad39d., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2 from cache 2024-11-13T16:11:26,705 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.FailedServers(52): Added failed server with address f8f85b0bb1d8:43467 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: f8f85b0bb1d8/172.17.0.3:43467 2024-11-13T16:11:26,807 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to f8f85b0bb1d8:43467 this server is in the failed servers list 2024-11-13T16:11:26,807 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=f8f85b0bb1d8:43467 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:43467 2024-11-13T16:11:26,807 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:43467 2024-11-13T16:11:26,807 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=-1 from cache 2024-11-13T16:11:26,916 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T16:11:26,918 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55792, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T16:11:26,919 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=-1] 2024-11-13T16:11:26,922 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='bbb', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731514091776.8ba796d80fc3b65f0948c95ccbdad39d., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=5] 2024-11-13T16:11:26,924 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47748, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T16:11:26,959 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to f8f85b0bb1d8:43467 this server is in the failed servers list 2024-11-13T16:11:26,960 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731514091776.3945ce860c8642051e17517944a83136., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731514091776.3945ce860c8642051e17517944a83136., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=f8f85b0bb1d8:43467 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:43467 2024-11-13T16:11:26,960 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731514091776.3945ce860c8642051e17517944a83136., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:43467 2024-11-13T16:11:26,960 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731514091776.3945ce860c8642051e17517944a83136., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2 from cache 2024-11-13T16:11:27,075 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='fff', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731514091776.3945ce860c8642051e17517944a83136., hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=5] 2024-11-13T16:11:27,106 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to f8f85b0bb1d8:43467 this server is in the failed servers list 2024-11-13T16:11:27,107 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731514091776.e8725a794e6d251e1bf6e3502baa7700., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731514091776.e8725a794e6d251e1bf6e3502baa7700., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=f8f85b0bb1d8:43467 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:43467 2024-11-13T16:11:27,107 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731514091776.e8725a794e6d251e1bf6e3502baa7700., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:43467 2024-11-13T16:11:27,107 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731514091776.e8725a794e6d251e1bf6e3502baa7700., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2 from cache 2024-11-13T16:11:27,227 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iii', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731514091776.e8725a794e6d251e1bf6e3502baa7700., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=5] 2024-11-13T16:11:27,273 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to f8f85b0bb1d8:43467 this server is in the failed servers list 2024-11-13T16:11:27,274 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731514091776.aa124b802c708ca1f715cb1f8c9c4eaa., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731514091776.aa124b802c708ca1f715cb1f8c9c4eaa., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=f8f85b0bb1d8:43467 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:43467 2024-11-13T16:11:27,274 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731514091776.aa124b802c708ca1f715cb1f8c9c4eaa., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:43467 2024-11-13T16:11:27,274 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731514091776.aa124b802c708ca1f715cb1f8c9c4eaa., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2 from cache 2024-11-13T16:11:27,392 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731514091776.aa124b802c708ca1f715cb1f8c9c4eaa., hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=5] 2024-11-13T16:11:27,422 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to f8f85b0bb1d8:43467 this server is in the failed servers list 2024-11-13T16:11:27,423 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731514091776.29df624bbcb878edce7fcdf0abceea88., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731514091776.29df624bbcb878edce7fcdf0abceea88., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=f8f85b0bb1d8:43467 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:43467 2024-11-13T16:11:27,423 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731514091776.29df624bbcb878edce7fcdf0abceea88., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:43467 2024-11-13T16:11:27,423 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731514091776.29df624bbcb878edce7fcdf0abceea88., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2 from cache 2024-11-13T16:11:27,536 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ooo', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731514091776.29df624bbcb878edce7fcdf0abceea88., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=5] 2024-11-13T16:11:27,555 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.AbstractRpcClient(357): Not trying to connect to f8f85b0bb1d8:43467 this server is in the failed servers list 2024-11-13T16:11:27,556 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731514091776.fec5f21b48c36bf01685ec2ee58f0490., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731514091776.fec5f21b48c36bf01685ec2ee58f0490., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=f8f85b0bb1d8:43467 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:43467 2024-11-13T16:11:27,556 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731514091776.fec5f21b48c36bf01685ec2ee58f0490., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:43467 2024-11-13T16:11:27,556 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731514091776.fec5f21b48c36bf01685ec2ee58f0490., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2 from cache 2024-11-13T16:11:27,672 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='sss', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731514091776.fec5f21b48c36bf01685ec2ee58f0490., hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=5] 2024-11-13T16:11:27,682 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to f8f85b0bb1d8:43467 this server is in the failed servers list 2024-11-13T16:11:27,682 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731514091776.519ece296a21b5ddccc38a7c928780bb., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731514091776.519ece296a21b5ddccc38a7c928780bb., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=f8f85b0bb1d8:43467 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:43467 2024-11-13T16:11:27,683 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731514091776.519ece296a21b5ddccc38a7c928780bb., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:43467 2024-11-13T16:11:27,683 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731514091776.519ece296a21b5ddccc38a7c928780bb., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2 from cache 2024-11-13T16:11:27,791 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uuu', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731514091776.519ece296a21b5ddccc38a7c928780bb., hostname=f8f85b0bb1d8,38187,1731514088725, seqNum=5] 2024-11-13T16:11:27,809 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to f8f85b0bb1d8:43467 this server is in the failed servers list 2024-11-13T16:11:27,809 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731514091776.6be98d350311c17b11fdb9e9a1238fca., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731514091776.6be98d350311c17b11fdb9e9a1238fca., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=f8f85b0bb1d8:43467 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:43467 2024-11-13T16:11:27,809 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731514091776.6be98d350311c17b11fdb9e9a1238fca., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:43467 2024-11-13T16:11:27,809 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731514091776.6be98d350311c17b11fdb9e9a1238fca., hostname=f8f85b0bb1d8,43467,1731514088560, seqNum=2 from cache 2024-11-13T16:11:27,921 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yyy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731514091776.6be98d350311c17b11fdb9e9a1238fca., hostname=f8f85b0bb1d8,41333,1731514088798, seqNum=5] 2024-11-13T16:11:28,022 WARN [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false 2024-11-13T16:11:28,022 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-13T16:11:28,022 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T16:11:28,022 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.procedure.TestSCPBase.tearDown(TestSCPBase.java:72) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T16:11:28,022 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T16:11:28,023 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T16:11:28,023 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T16:11:28,023 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-13T16:11:28,023 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1965503347, stopped=false 2024-11-13T16:11:28,023 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=f8f85b0bb1d8,42949,1731514087490 2024-11-13T16:11:28,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41333-0x1013621fa650003, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T16:11:28,113 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38187-0x1013621fa650002, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T16:11:28,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41333-0x1013621fa650003, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:11:28,113 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38187-0x1013621fa650002, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:11:28,113 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T16:11:28,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42949-0x1013621fa650000, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T16:11:28,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42949-0x1013621fa650000, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:11:28,113 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T16:11:28,113 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.procedure.TestSCPBase.tearDown(TestSCPBase.java:72) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T16:11:28,113 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T16:11:28,114 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f8f85b0bb1d8,38187,1731514088725' ***** 2024-11-13T16:11:28,114 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T16:11:28,114 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41333-0x1013621fa650003, quorum=127.0.0.1:60455, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T16:11:28,114 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38187-0x1013621fa650002, quorum=127.0.0.1:60455, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T16:11:28,114 INFO [RS:1;f8f85b0bb1d8:38187 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T16:11:28,114 INFO [RS:1;f8f85b0bb1d8:38187 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T16:11:28,114 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T16:11:28,114 INFO [RS:1;f8f85b0bb1d8:38187 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T16:11:28,114 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f8f85b0bb1d8,41333,1731514088798' ***** 2024-11-13T16:11:28,114 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T16:11:28,114 INFO [RS:1;f8f85b0bb1d8:38187 {}] regionserver.HRegionServer(3091): Received CLOSE for 7c34e9ded13254b91545186c85befbf8 2024-11-13T16:11:28,115 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42949-0x1013621fa650000, quorum=127.0.0.1:60455, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T16:11:28,115 INFO [RS:2;f8f85b0bb1d8:41333 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T16:11:28,115 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T16:11:28,115 INFO [RS:2;f8f85b0bb1d8:41333 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T16:11:28,115 INFO [RS:2;f8f85b0bb1d8:41333 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T16:11:28,115 INFO [RS:2;f8f85b0bb1d8:41333 {}] regionserver.HRegionServer(3091): Received CLOSE for fe97fe0e9b37811ba99ca8100b461589 2024-11-13T16:11:28,115 INFO [RS:1;f8f85b0bb1d8:38187 {}] regionserver.HRegionServer(3091): Received CLOSE for 4a146f98682f25e8cddcae5dbf318603 2024-11-13T16:11:28,115 INFO [RS:1;f8f85b0bb1d8:38187 {}] regionserver.HRegionServer(3091): Received CLOSE for 78285ca993cc7fbfd78041c30558f756 2024-11-13T16:11:28,115 INFO [RS:1;f8f85b0bb1d8:38187 {}] regionserver.HRegionServer(3091): Received CLOSE for 97998e7179d85b23a3f5928cd6511629 2024-11-13T16:11:28,115 INFO [RS:1;f8f85b0bb1d8:38187 {}] regionserver.HRegionServer(3091): Received CLOSE for 3647edbd8aef23dda3887b61277da1d3 2024-11-13T16:11:28,115 INFO [RS:1;f8f85b0bb1d8:38187 {}] regionserver.HRegionServer(3091): Received CLOSE for 34f3d3b6a6dae1f4077567bed0bd8e75 2024-11-13T16:11:28,115 INFO [RS:1;f8f85b0bb1d8:38187 {}] regionserver.HRegionServer(3091): Received CLOSE for 93f8ed38b68bd013eb0cf0e7615013ae 2024-11-13T16:11:28,115 INFO [RS:1;f8f85b0bb1d8:38187 {}] regionserver.HRegionServer(3091): Received CLOSE for 519ece296a21b5ddccc38a7c928780bb 2024-11-13T16:11:28,116 INFO [RS:1;f8f85b0bb1d8:38187 {}] regionserver.HRegionServer(3091): Received CLOSE for 8ba796d80fc3b65f0948c95ccbdad39d 2024-11-13T16:11:28,116 INFO [RS:1;f8f85b0bb1d8:38187 {}] regionserver.HRegionServer(3091): Received CLOSE for 29df624bbcb878edce7fcdf0abceea88 2024-11-13T16:11:28,116 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7c34e9ded13254b91545186c85befbf8, disabling compactions & flushes 2024-11-13T16:11:28,116 INFO [RS:1;f8f85b0bb1d8:38187 {}] regionserver.HRegionServer(3091): Received CLOSE for c4a87adf770b37d40ab6691e377ad1b5 2024-11-13T16:11:28,116 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731514091776.7c34e9ded13254b91545186c85befbf8. 2024-11-13T16:11:28,116 INFO [RS:1;f8f85b0bb1d8:38187 {}] regionserver.HRegionServer(3091): Received CLOSE for 1a610c2403a0b491d368473a6e1fb049 2024-11-13T16:11:28,116 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731514091776.7c34e9ded13254b91545186c85befbf8. 2024-11-13T16:11:28,116 INFO [RS:1;f8f85b0bb1d8:38187 {}] regionserver.HRegionServer(3091): Received CLOSE for e8725a794e6d251e1bf6e3502baa7700 2024-11-13T16:11:28,116 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731514091776.7c34e9ded13254b91545186c85befbf8. after waiting 0 ms 2024-11-13T16:11:28,116 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731514091776.7c34e9ded13254b91545186c85befbf8. 2024-11-13T16:11:28,116 INFO [RS:1;f8f85b0bb1d8:38187 {}] regionserver.HRegionServer(959): stopping server f8f85b0bb1d8,38187,1731514088725 2024-11-13T16:11:28,116 INFO [RS:1;f8f85b0bb1d8:38187 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T16:11:28,116 INFO [RS:1;f8f85b0bb1d8:38187 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;f8f85b0bb1d8:38187. 2024-11-13T16:11:28,116 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 7c34e9ded13254b91545186c85befbf8 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T16:11:28,116 DEBUG [RS:1;f8f85b0bb1d8:38187 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T16:11:28,116 DEBUG [RS:1;f8f85b0bb1d8:38187 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T16:11:28,117 INFO [RS:1;f8f85b0bb1d8:38187 {}] regionserver.HRegionServer(1321): Waiting on 13 regions to close 2024-11-13T16:11:28,117 DEBUG [RS:1;f8f85b0bb1d8:38187 {}] regionserver.HRegionServer(1351): Waiting on 1a610c2403a0b491d368473a6e1fb049, 29df624bbcb878edce7fcdf0abceea88, 34f3d3b6a6dae1f4077567bed0bd8e75, 3647edbd8aef23dda3887b61277da1d3, 4a146f98682f25e8cddcae5dbf318603, 519ece296a21b5ddccc38a7c928780bb, 78285ca993cc7fbfd78041c30558f756, 7c34e9ded13254b91545186c85befbf8, 8ba796d80fc3b65f0948c95ccbdad39d, 93f8ed38b68bd013eb0cf0e7615013ae, 97998e7179d85b23a3f5928cd6511629, c4a87adf770b37d40ab6691e377ad1b5, e8725a794e6d251e1bf6e3502baa7700 2024-11-13T16:11:28,118 INFO [RS:2;f8f85b0bb1d8:41333 {}] regionserver.HRegionServer(3091): Received CLOSE for fec5f21b48c36bf01685ec2ee58f0490 2024-11-13T16:11:28,119 INFO [RS:2;f8f85b0bb1d8:41333 {}] regionserver.HRegionServer(3091): Received CLOSE for bf1f9f9bc1bc9ffce566d41f8ae3619a 2024-11-13T16:11:28,119 INFO [RS:2;f8f85b0bb1d8:41333 {}] regionserver.HRegionServer(3091): Received CLOSE for fa6636be3559bc84793534420cd4f207 2024-11-13T16:11:28,119 INFO [RS:2;f8f85b0bb1d8:41333 {}] regionserver.HRegionServer(3091): Received CLOSE for 72d273770314688e78624993de4f3398 2024-11-13T16:11:28,119 INFO [RS:2;f8f85b0bb1d8:41333 {}] regionserver.HRegionServer(3091): Received CLOSE for 3945ce860c8642051e17517944a83136 2024-11-13T16:11:28,119 INFO [RS:2;f8f85b0bb1d8:41333 {}] regionserver.HRegionServer(3091): Received CLOSE for 6be98d350311c17b11fdb9e9a1238fca 2024-11-13T16:11:28,119 INFO [RS:2;f8f85b0bb1d8:41333 {}] regionserver.HRegionServer(3091): Received CLOSE for 4aaf5667f132c289386d1b3b0c0ac57f 2024-11-13T16:11:28,119 INFO [RS:2;f8f85b0bb1d8:41333 {}] regionserver.HRegionServer(3091): Received CLOSE for aa124b802c708ca1f715cb1f8c9c4eaa 2024-11-13T16:11:28,119 INFO [RS:2;f8f85b0bb1d8:41333 {}] regionserver.HRegionServer(3091): Received CLOSE for 1fcf78578ccab75f1705f3c2d7bcdb14 2024-11-13T16:11:28,119 INFO [RS:2;f8f85b0bb1d8:41333 {}] regionserver.HRegionServer(3091): Received CLOSE for 05d65f4a6bfecb3d530d15832c218252 2024-11-13T16:11:28,119 INFO [RS:2;f8f85b0bb1d8:41333 {}] regionserver.HRegionServer(3091): Received CLOSE for e72e57b96ff9bc0bbd862ce564d79c42 2024-11-13T16:11:28,119 INFO [RS:2;f8f85b0bb1d8:41333 {}] regionserver.HRegionServer(3091): Received CLOSE for f8f8b8e4ae8470fd8a1d2bf0684947ef 2024-11-13T16:11:28,119 INFO [RS:2;f8f85b0bb1d8:41333 {}] regionserver.HRegionServer(959): stopping server f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:11:28,119 INFO [RS:2;f8f85b0bb1d8:41333 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T16:11:28,119 INFO [RS:2;f8f85b0bb1d8:41333 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;f8f85b0bb1d8:41333. 2024-11-13T16:11:28,119 DEBUG [RS:2;f8f85b0bb1d8:41333 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T16:11:28,119 DEBUG [RS:2;f8f85b0bb1d8:41333 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T16:11:28,120 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fe97fe0e9b37811ba99ca8100b461589, disabling compactions & flushes 2024-11-13T16:11:28,120 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731514091776.fe97fe0e9b37811ba99ca8100b461589. 2024-11-13T16:11:28,120 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731514091776.fe97fe0e9b37811ba99ca8100b461589. 2024-11-13T16:11:28,120 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731514091776.fe97fe0e9b37811ba99ca8100b461589. after waiting 0 ms 2024-11-13T16:11:28,120 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731514091776.fe97fe0e9b37811ba99ca8100b461589. 2024-11-13T16:11:28,120 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing fe97fe0e9b37811ba99ca8100b461589 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T16:11:28,120 INFO [RS:2;f8f85b0bb1d8:41333 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T16:11:28,120 INFO [RS:2;f8f85b0bb1d8:41333 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T16:11:28,120 INFO [RS:2;f8f85b0bb1d8:41333 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T16:11:28,120 INFO [RS:2;f8f85b0bb1d8:41333 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-13T16:11:28,123 INFO [RS:2;f8f85b0bb1d8:41333 {}] regionserver.HRegionServer(1321): Waiting on 14 regions to close 2024-11-13T16:11:28,124 DEBUG [RS:2;f8f85b0bb1d8:41333 {}] regionserver.HRegionServer(1351): Waiting on 05d65f4a6bfecb3d530d15832c218252, 1588230740, 1fcf78578ccab75f1705f3c2d7bcdb14, 3945ce860c8642051e17517944a83136, 4aaf5667f132c289386d1b3b0c0ac57f, 6be98d350311c17b11fdb9e9a1238fca, 72d273770314688e78624993de4f3398, aa124b802c708ca1f715cb1f8c9c4eaa, bf1f9f9bc1bc9ffce566d41f8ae3619a, e72e57b96ff9bc0bbd862ce564d79c42, f8f8b8e4ae8470fd8a1d2bf0684947ef, fa6636be3559bc84793534420cd4f207, fe97fe0e9b37811ba99ca8100b461589, fec5f21b48c36bf01685ec2ee58f0490 2024-11-13T16:11:28,127 DEBUG [RS_CLOSE_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T16:11:28,127 INFO [RS_CLOSE_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T16:11:28,127 DEBUG [RS_CLOSE_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T16:11:28,127 DEBUG [RS_CLOSE_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T16:11:28,127 DEBUG [RS_CLOSE_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T16:11:28,128 INFO [RS_CLOSE_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=11.91 KB heapSize=17.63 KB 2024-11-13T16:11:28,135 INFO [regionserver/f8f85b0bb1d8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T16:11:28,149 DEBUG [RS_CLOSE_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/hbase/meta/1588230740/.tmp/info/92dd2a68fc2145db8509bd5d31152bcb is 268, key is testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731514091776.8ba796d80fc3b65f0948c95ccbdad39d./info:regioninfo/1731514284599/Put/seqid=0 2024-11-13T16:11:28,157 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7c34e9ded13254b91545186c85befbf8/.tmp/colfamily11/8ac3189789e9489f8cf4b8d614a73797 is 52, key is qqq/colfamily11:colfamily11/1731514098791/Put/seqid=0 2024-11-13T16:11:28,159 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fe97fe0e9b37811ba99ca8100b461589/.tmp/colfamily11/d7fefb672b5e467f87deda3a17901392 is 52, key is ppp/colfamily11:colfamily11/1731514098535/Put/seqid=0 2024-11-13T16:11:28,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741885_1061 (size=18322) 2024-11-13T16:11:28,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741885_1061 (size=18322) 2024-11-13T16:11:28,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741885_1061 (size=18322) 2024-11-13T16:11:28,197 INFO [regionserver/f8f85b0bb1d8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T16:11:28,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741886_1062 (size=39263) 2024-11-13T16:11:28,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741886_1062 (size=39263) 2024-11-13T16:11:28,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741886_1062 (size=39263) 2024-11-13T16:11:28,210 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7c34e9ded13254b91545186c85befbf8/.tmp/colfamily11/8ac3189789e9489f8cf4b8d614a73797 2024-11-13T16:11:28,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741887_1063 (size=39263) 2024-11-13T16:11:28,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741887_1063 (size=39263) 2024-11-13T16:11:28,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741887_1063 (size=39263) 2024-11-13T16:11:28,216 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fe97fe0e9b37811ba99ca8100b461589/.tmp/colfamily11/d7fefb672b5e467f87deda3a17901392 2024-11-13T16:11:28,219 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7c34e9ded13254b91545186c85befbf8/.tmp/colfamily11/8ac3189789e9489f8cf4b8d614a73797 as hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7c34e9ded13254b91545186c85befbf8/colfamily11/8ac3189789e9489f8cf4b8d614a73797 2024-11-13T16:11:28,224 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fe97fe0e9b37811ba99ca8100b461589/.tmp/colfamily11/d7fefb672b5e467f87deda3a17901392 as hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fe97fe0e9b37811ba99ca8100b461589/colfamily11/d7fefb672b5e467f87deda3a17901392 2024-11-13T16:11:28,227 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7c34e9ded13254b91545186c85befbf8/colfamily11/8ac3189789e9489f8cf4b8d614a73797, entries=703, sequenceid=5, filesize=38.3 K 2024-11-13T16:11:28,228 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 7c34e9ded13254b91545186c85befbf8 in 111ms, sequenceid=5, compaction requested=false 2024-11-13T16:11:28,230 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fe97fe0e9b37811ba99ca8100b461589/colfamily11/d7fefb672b5e467f87deda3a17901392, entries=703, sequenceid=5, filesize=38.3 K 2024-11-13T16:11:28,230 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for fe97fe0e9b37811ba99ca8100b461589 in 110ms, sequenceid=5, compaction requested=false 2024-11-13T16:11:28,238 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7c34e9ded13254b91545186c85befbf8/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T16:11:28,238 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fe97fe0e9b37811ba99ca8100b461589/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T16:11:28,241 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731514091776.7c34e9ded13254b91545186c85befbf8. 2024-11-13T16:11:28,241 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731514091776.fe97fe0e9b37811ba99ca8100b461589. 2024-11-13T16:11:28,241 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7c34e9ded13254b91545186c85befbf8: Waiting for close lock at 1731514288115Running coprocessor pre-close hooks at 1731514288116 (+1 ms)Disabling compacts and flushes for region at 1731514288116Disabling writes for close at 1731514288116Obtaining lock to block concurrent updates at 1731514288116Preparing flush snapshotting stores in 7c34e9ded13254b91545186c85befbf8 at 1731514288116Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731514091776.7c34e9ded13254b91545186c85befbf8., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731514288122 (+6 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731514091776.7c34e9ded13254b91545186c85befbf8. at 1731514288127 (+5 ms)Flushing 7c34e9ded13254b91545186c85befbf8/colfamily11: creating writer at 1731514288127Flushing 7c34e9ded13254b91545186c85befbf8/colfamily11: appending metadata at 1731514288157 (+30 ms)Flushing 7c34e9ded13254b91545186c85befbf8/colfamily11: closing flushed file at 1731514288157Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59c2698b: reopening flushed file at 1731514288218 (+61 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 7c34e9ded13254b91545186c85befbf8 in 111ms, sequenceid=5, compaction requested=false at 1731514288228 (+10 ms)Writing region close event to WAL at 1731514288234 (+6 ms)Running coprocessor post-close hooks at 1731514288238 (+4 ms)Closed at 1731514288240 (+2 ms) 2024-11-13T16:11:28,241 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fe97fe0e9b37811ba99ca8100b461589: Waiting for close lock at 1731514288119Running coprocessor pre-close hooks at 1731514288119Disabling compacts and flushes for region at 1731514288119Disabling writes for close at 1731514288120 (+1 ms)Obtaining lock to block concurrent updates at 1731514288120Preparing flush snapshotting stores in fe97fe0e9b37811ba99ca8100b461589 at 1731514288120Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731514091776.fe97fe0e9b37811ba99ca8100b461589., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731514288122 (+2 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731514091776.fe97fe0e9b37811ba99ca8100b461589. at 1731514288131 (+9 ms)Flushing fe97fe0e9b37811ba99ca8100b461589/colfamily11: creating writer at 1731514288131Flushing fe97fe0e9b37811ba99ca8100b461589/colfamily11: appending metadata at 1731514288158 (+27 ms)Flushing fe97fe0e9b37811ba99ca8100b461589/colfamily11: closing flushed file at 1731514288158Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9f39963: reopening flushed file at 1731514288223 (+65 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for fe97fe0e9b37811ba99ca8100b461589 in 110ms, sequenceid=5, compaction requested=false at 1731514288231 (+8 ms)Writing region close event to WAL at 1731514288234 (+3 ms)Running coprocessor post-close hooks at 1731514288238 (+4 ms)Closed at 1731514288240 (+2 ms) 2024-11-13T16:11:28,241 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731514091776.fe97fe0e9b37811ba99ca8100b461589. 2024-11-13T16:11:28,241 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731514091776.7c34e9ded13254b91545186c85befbf8. 2024-11-13T16:11:28,241 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fec5f21b48c36bf01685ec2ee58f0490, disabling compactions & flushes 2024-11-13T16:11:28,241 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4a146f98682f25e8cddcae5dbf318603, disabling compactions & flushes 2024-11-13T16:11:28,241 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731514091776.fec5f21b48c36bf01685ec2ee58f0490. 2024-11-13T16:11:28,241 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731514091776.4a146f98682f25e8cddcae5dbf318603. 2024-11-13T16:11:28,241 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731514091776.fec5f21b48c36bf01685ec2ee58f0490. 2024-11-13T16:11:28,241 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731514091776.4a146f98682f25e8cddcae5dbf318603. 2024-11-13T16:11:28,241 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731514091776.fec5f21b48c36bf01685ec2ee58f0490. after waiting 0 ms 2024-11-13T16:11:28,241 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731514091776.fec5f21b48c36bf01685ec2ee58f0490. 2024-11-13T16:11:28,241 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731514091776.4a146f98682f25e8cddcae5dbf318603. after waiting 0 ms 2024-11-13T16:11:28,241 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731514091776.4a146f98682f25e8cddcae5dbf318603. 2024-11-13T16:11:28,242 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 4a146f98682f25e8cddcae5dbf318603 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T16:11:28,248 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fec5f21b48c36bf01685ec2ee58f0490/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-13T16:11:28,249 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731514091776.fec5f21b48c36bf01685ec2ee58f0490. 2024-11-13T16:11:28,249 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fec5f21b48c36bf01685ec2ee58f0490: Waiting for close lock at 1731514288241Running coprocessor pre-close hooks at 1731514288241Disabling compacts and flushes for region at 1731514288241Disabling writes for close at 1731514288241Writing region close event to WAL at 1731514288244 (+3 ms)Running coprocessor post-close hooks at 1731514288249 (+5 ms)Closed at 1731514288249 2024-11-13T16:11:28,249 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731514091776.fec5f21b48c36bf01685ec2ee58f0490. 2024-11-13T16:11:28,250 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing bf1f9f9bc1bc9ffce566d41f8ae3619a, disabling compactions & flushes 2024-11-13T16:11:28,250 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731514091776.bf1f9f9bc1bc9ffce566d41f8ae3619a. 2024-11-13T16:11:28,250 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731514091776.bf1f9f9bc1bc9ffce566d41f8ae3619a. 2024-11-13T16:11:28,250 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731514091776.bf1f9f9bc1bc9ffce566d41f8ae3619a. after waiting 0 ms 2024-11-13T16:11:28,250 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731514091776.bf1f9f9bc1bc9ffce566d41f8ae3619a. 2024-11-13T16:11:28,250 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing bf1f9f9bc1bc9ffce566d41f8ae3619a 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T16:11:28,266 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4a146f98682f25e8cddcae5dbf318603/.tmp/colfamily11/2cffa98180e9496dbab944cfe53d745c is 52, key is xxx/colfamily11:colfamily11/1731514098914/Put/seqid=0 2024-11-13T16:11:28,274 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bf1f9f9bc1bc9ffce566d41f8ae3619a/.tmp/colfamily11/7c50cfc42e0c40e9b3a02a251bab64ea is 52, key is www/colfamily11:colfamily11/1731514098619/Put/seqid=0 2024-11-13T16:11:28,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741888_1064 (size=39263) 2024-11-13T16:11:28,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741888_1064 (size=39263) 2024-11-13T16:11:28,295 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4a146f98682f25e8cddcae5dbf318603/.tmp/colfamily11/2cffa98180e9496dbab944cfe53d745c 2024-11-13T16:11:28,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741888_1064 (size=39263) 2024-11-13T16:11:28,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741889_1065 (size=39263) 2024-11-13T16:11:28,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741889_1065 (size=39263) 2024-11-13T16:11:28,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741889_1065 (size=39263) 2024-11-13T16:11:28,299 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bf1f9f9bc1bc9ffce566d41f8ae3619a/.tmp/colfamily11/7c50cfc42e0c40e9b3a02a251bab64ea 2024-11-13T16:11:28,305 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4a146f98682f25e8cddcae5dbf318603/.tmp/colfamily11/2cffa98180e9496dbab944cfe53d745c as hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4a146f98682f25e8cddcae5dbf318603/colfamily11/2cffa98180e9496dbab944cfe53d745c 2024-11-13T16:11:28,309 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bf1f9f9bc1bc9ffce566d41f8ae3619a/.tmp/colfamily11/7c50cfc42e0c40e9b3a02a251bab64ea as hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bf1f9f9bc1bc9ffce566d41f8ae3619a/colfamily11/7c50cfc42e0c40e9b3a02a251bab64ea 2024-11-13T16:11:28,315 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4a146f98682f25e8cddcae5dbf318603/colfamily11/2cffa98180e9496dbab944cfe53d745c, entries=703, sequenceid=5, filesize=38.3 K 2024-11-13T16:11:28,317 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bf1f9f9bc1bc9ffce566d41f8ae3619a/colfamily11/7c50cfc42e0c40e9b3a02a251bab64ea, entries=703, sequenceid=5, filesize=38.3 K 2024-11-13T16:11:28,318 DEBUG [RS:1;f8f85b0bb1d8:38187 {}] regionserver.HRegionServer(1351): Waiting on 1a610c2403a0b491d368473a6e1fb049, 29df624bbcb878edce7fcdf0abceea88, 34f3d3b6a6dae1f4077567bed0bd8e75, 3647edbd8aef23dda3887b61277da1d3, 4a146f98682f25e8cddcae5dbf318603, 519ece296a21b5ddccc38a7c928780bb, 78285ca993cc7fbfd78041c30558f756, 8ba796d80fc3b65f0948c95ccbdad39d, 93f8ed38b68bd013eb0cf0e7615013ae, 97998e7179d85b23a3f5928cd6511629, c4a87adf770b37d40ab6691e377ad1b5, e8725a794e6d251e1bf6e3502baa7700 2024-11-13T16:11:28,318 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for bf1f9f9bc1bc9ffce566d41f8ae3619a in 68ms, sequenceid=5, compaction requested=false 2024-11-13T16:11:28,319 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 4a146f98682f25e8cddcae5dbf318603 in 76ms, sequenceid=5, compaction requested=false 2024-11-13T16:11:28,324 DEBUG [RS:2;f8f85b0bb1d8:41333 {}] regionserver.HRegionServer(1351): Waiting on 05d65f4a6bfecb3d530d15832c218252, 1588230740, 1fcf78578ccab75f1705f3c2d7bcdb14, 3945ce860c8642051e17517944a83136, 4aaf5667f132c289386d1b3b0c0ac57f, 6be98d350311c17b11fdb9e9a1238fca, 72d273770314688e78624993de4f3398, aa124b802c708ca1f715cb1f8c9c4eaa, bf1f9f9bc1bc9ffce566d41f8ae3619a, e72e57b96ff9bc0bbd862ce564d79c42, f8f8b8e4ae8470fd8a1d2bf0684947ef, fa6636be3559bc84793534420cd4f207 2024-11-13T16:11:28,328 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bf1f9f9bc1bc9ffce566d41f8ae3619a/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T16:11:28,329 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731514091776.bf1f9f9bc1bc9ffce566d41f8ae3619a. 2024-11-13T16:11:28,329 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for bf1f9f9bc1bc9ffce566d41f8ae3619a: Waiting for close lock at 1731514288250Running coprocessor pre-close hooks at 1731514288250Disabling compacts and flushes for region at 1731514288250Disabling writes for close at 1731514288250Obtaining lock to block concurrent updates at 1731514288250Preparing flush snapshotting stores in bf1f9f9bc1bc9ffce566d41f8ae3619a at 1731514288250Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731514091776.bf1f9f9bc1bc9ffce566d41f8ae3619a., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731514288250Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731514091776.bf1f9f9bc1bc9ffce566d41f8ae3619a. at 1731514288251 (+1 ms)Flushing bf1f9f9bc1bc9ffce566d41f8ae3619a/colfamily11: creating writer at 1731514288251Flushing bf1f9f9bc1bc9ffce566d41f8ae3619a/colfamily11: appending metadata at 1731514288274 (+23 ms)Flushing bf1f9f9bc1bc9ffce566d41f8ae3619a/colfamily11: closing flushed file at 1731514288274Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ed1e7d2: reopening flushed file at 1731514288308 (+34 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for bf1f9f9bc1bc9ffce566d41f8ae3619a in 68ms, sequenceid=5, compaction requested=false at 1731514288318 (+10 ms)Writing region close event to WAL at 1731514288324 (+6 ms)Running coprocessor post-close hooks at 1731514288329 (+5 ms)Closed at 1731514288329 2024-11-13T16:11:28,330 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731514091776.bf1f9f9bc1bc9ffce566d41f8ae3619a. 2024-11-13T16:11:28,330 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fa6636be3559bc84793534420cd4f207, disabling compactions & flushes 2024-11-13T16:11:28,330 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731514091776.fa6636be3559bc84793534420cd4f207. 2024-11-13T16:11:28,330 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731514091776.fa6636be3559bc84793534420cd4f207. 2024-11-13T16:11:28,330 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731514091776.fa6636be3559bc84793534420cd4f207. after waiting 0 ms 2024-11-13T16:11:28,330 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731514091776.fa6636be3559bc84793534420cd4f207. 2024-11-13T16:11:28,330 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing fa6636be3559bc84793534420cd4f207 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T16:11:28,335 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4a146f98682f25e8cddcae5dbf318603/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T16:11:28,336 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731514091776.4a146f98682f25e8cddcae5dbf318603. 2024-11-13T16:11:28,336 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4a146f98682f25e8cddcae5dbf318603: Waiting for close lock at 1731514288241Running coprocessor pre-close hooks at 1731514288241Disabling compacts and flushes for region at 1731514288241Disabling writes for close at 1731514288241Obtaining lock to block concurrent updates at 1731514288242 (+1 ms)Preparing flush snapshotting stores in 4a146f98682f25e8cddcae5dbf318603 at 1731514288242Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731514091776.4a146f98682f25e8cddcae5dbf318603., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731514288242Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731514091776.4a146f98682f25e8cddcae5dbf318603. at 1731514288243 (+1 ms)Flushing 4a146f98682f25e8cddcae5dbf318603/colfamily11: creating writer at 1731514288243Flushing 4a146f98682f25e8cddcae5dbf318603/colfamily11: appending metadata at 1731514288266 (+23 ms)Flushing 4a146f98682f25e8cddcae5dbf318603/colfamily11: closing flushed file at 1731514288266Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3be6b768: reopening flushed file at 1731514288303 (+37 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 4a146f98682f25e8cddcae5dbf318603 in 76ms, sequenceid=5, compaction requested=false at 1731514288319 (+16 ms)Writing region close event to WAL at 1731514288323 (+4 ms)Running coprocessor post-close hooks at 1731514288336 (+13 ms)Closed at 1731514288336 2024-11-13T16:11:28,336 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731514091776.4a146f98682f25e8cddcae5dbf318603. 2024-11-13T16:11:28,336 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 78285ca993cc7fbfd78041c30558f756, disabling compactions & flushes 2024-11-13T16:11:28,336 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731514091776.78285ca993cc7fbfd78041c30558f756. 2024-11-13T16:11:28,336 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731514091776.78285ca993cc7fbfd78041c30558f756. 2024-11-13T16:11:28,336 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731514091776.78285ca993cc7fbfd78041c30558f756. after waiting 0 ms 2024-11-13T16:11:28,336 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731514091776.78285ca993cc7fbfd78041c30558f756. 2024-11-13T16:11:28,337 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 78285ca993cc7fbfd78041c30558f756 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T16:11:28,355 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fa6636be3559bc84793534420cd4f207/.tmp/colfamily11/0fffe3e1659d4fc0be86365a4c0b1489 is 52, key is jjj/colfamily11:colfamily11/1731514098430/Put/seqid=0 2024-11-13T16:11:28,360 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/78285ca993cc7fbfd78041c30558f756/.tmp/colfamily11/e5ab851ad0b144869e93c389f30ee9cc is 52, key is lll/colfamily11:colfamily11/1731514098694/Put/seqid=0 2024-11-13T16:11:28,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741890_1066 (size=39263) 2024-11-13T16:11:28,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741890_1066 (size=39263) 2024-11-13T16:11:28,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741890_1066 (size=39263) 2024-11-13T16:11:28,363 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fa6636be3559bc84793534420cd4f207/.tmp/colfamily11/0fffe3e1659d4fc0be86365a4c0b1489 2024-11-13T16:11:28,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741891_1067 (size=39263) 2024-11-13T16:11:28,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741891_1067 (size=39263) 2024-11-13T16:11:28,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741891_1067 (size=39263) 2024-11-13T16:11:28,370 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/78285ca993cc7fbfd78041c30558f756/.tmp/colfamily11/e5ab851ad0b144869e93c389f30ee9cc 2024-11-13T16:11:28,371 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fa6636be3559bc84793534420cd4f207/.tmp/colfamily11/0fffe3e1659d4fc0be86365a4c0b1489 as hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fa6636be3559bc84793534420cd4f207/colfamily11/0fffe3e1659d4fc0be86365a4c0b1489 2024-11-13T16:11:28,381 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/78285ca993cc7fbfd78041c30558f756/.tmp/colfamily11/e5ab851ad0b144869e93c389f30ee9cc as hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/78285ca993cc7fbfd78041c30558f756/colfamily11/e5ab851ad0b144869e93c389f30ee9cc 2024-11-13T16:11:28,384 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fa6636be3559bc84793534420cd4f207/colfamily11/0fffe3e1659d4fc0be86365a4c0b1489, entries=703, sequenceid=5, filesize=38.3 K 2024-11-13T16:11:28,385 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for fa6636be3559bc84793534420cd4f207 in 55ms, sequenceid=5, compaction requested=false 2024-11-13T16:11:28,391 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fa6636be3559bc84793534420cd4f207/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T16:11:28,392 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/78285ca993cc7fbfd78041c30558f756/colfamily11/e5ab851ad0b144869e93c389f30ee9cc, entries=703, sequenceid=5, filesize=38.3 K 2024-11-13T16:11:28,392 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731514091776.fa6636be3559bc84793534420cd4f207. 2024-11-13T16:11:28,392 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fa6636be3559bc84793534420cd4f207: Waiting for close lock at 1731514288330Running coprocessor pre-close hooks at 1731514288330Disabling compacts and flushes for region at 1731514288330Disabling writes for close at 1731514288330Obtaining lock to block concurrent updates at 1731514288330Preparing flush snapshotting stores in fa6636be3559bc84793534420cd4f207 at 1731514288330Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731514091776.fa6636be3559bc84793534420cd4f207., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731514288331 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731514091776.fa6636be3559bc84793534420cd4f207. at 1731514288332 (+1 ms)Flushing fa6636be3559bc84793534420cd4f207/colfamily11: creating writer at 1731514288332Flushing fa6636be3559bc84793534420cd4f207/colfamily11: appending metadata at 1731514288355 (+23 ms)Flushing fa6636be3559bc84793534420cd4f207/colfamily11: closing flushed file at 1731514288355Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@646c4e56: reopening flushed file at 1731514288370 (+15 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for fa6636be3559bc84793534420cd4f207 in 55ms, sequenceid=5, compaction requested=false at 1731514288385 (+15 ms)Writing region close event to WAL at 1731514288386 (+1 ms)Running coprocessor post-close hooks at 1731514288392 (+6 ms)Closed at 1731514288392 2024-11-13T16:11:28,392 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731514091776.fa6636be3559bc84793534420cd4f207. 2024-11-13T16:11:28,392 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 72d273770314688e78624993de4f3398, disabling compactions & flushes 2024-11-13T16:11:28,392 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731514091776.72d273770314688e78624993de4f3398. 2024-11-13T16:11:28,393 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731514091776.72d273770314688e78624993de4f3398. 2024-11-13T16:11:28,393 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731514091776.72d273770314688e78624993de4f3398. after waiting 0 ms 2024-11-13T16:11:28,393 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731514091776.72d273770314688e78624993de4f3398. 2024-11-13T16:11:28,393 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 72d273770314688e78624993de4f3398 3/3 column families, dataSize=48 B heapSize=888 B 2024-11-13T16:11:28,393 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 78285ca993cc7fbfd78041c30558f756 in 57ms, sequenceid=5, compaction requested=false 2024-11-13T16:11:28,397 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/78285ca993cc7fbfd78041c30558f756/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T16:11:28,398 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731514091776.78285ca993cc7fbfd78041c30558f756. 2024-11-13T16:11:28,398 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 78285ca993cc7fbfd78041c30558f756: Waiting for close lock at 1731514288336Running coprocessor pre-close hooks at 1731514288336Disabling compacts and flushes for region at 1731514288336Disabling writes for close at 1731514288336Obtaining lock to block concurrent updates at 1731514288337 (+1 ms)Preparing flush snapshotting stores in 78285ca993cc7fbfd78041c30558f756 at 1731514288337Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731514091776.78285ca993cc7fbfd78041c30558f756., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731514288337Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731514091776.78285ca993cc7fbfd78041c30558f756. at 1731514288339 (+2 ms)Flushing 78285ca993cc7fbfd78041c30558f756/colfamily11: creating writer at 1731514288339Flushing 78285ca993cc7fbfd78041c30558f756/colfamily11: appending metadata at 1731514288360 (+21 ms)Flushing 78285ca993cc7fbfd78041c30558f756/colfamily11: closing flushed file at 1731514288360Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@44f50c0c: reopening flushed file at 1731514288380 (+20 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 78285ca993cc7fbfd78041c30558f756 in 57ms, sequenceid=5, compaction requested=false at 1731514288393 (+13 ms)Writing region close event to WAL at 1731514288394 (+1 ms)Running coprocessor post-close hooks at 1731514288398 (+4 ms)Closed at 1731514288398 2024-11-13T16:11:28,398 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731514091776.78285ca993cc7fbfd78041c30558f756. 2024-11-13T16:11:28,399 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 97998e7179d85b23a3f5928cd6511629, disabling compactions & flushes 2024-11-13T16:11:28,399 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731514091776.97998e7179d85b23a3f5928cd6511629. 2024-11-13T16:11:28,399 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731514091776.97998e7179d85b23a3f5928cd6511629. 2024-11-13T16:11:28,399 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731514091776.97998e7179d85b23a3f5928cd6511629. after waiting 0 ms 2024-11-13T16:11:28,399 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731514091776.97998e7179d85b23a3f5928cd6511629. 2024-11-13T16:11:28,399 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 97998e7179d85b23a3f5928cd6511629 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T16:11:28,411 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/72d273770314688e78624993de4f3398/.tmp/colfamily11/f70a6fd3e50a43faa0ec678d364a1820 is 52, key is zzz/colfamily11:colfamily11/1731514098639/Put/seqid=0 2024-11-13T16:11:28,425 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/97998e7179d85b23a3f5928cd6511629/.tmp/colfamily11/52e218e5977c4530941e019a4d0569b7 is 52, key is ttt/colfamily11:colfamily11/1731514098827/Put/seqid=0 2024-11-13T16:11:28,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741892_1068 (size=4857) 2024-11-13T16:11:28,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741892_1068 (size=4857) 2024-11-13T16:11:28,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741892_1068 (size=4857) 2024-11-13T16:11:28,430 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/72d273770314688e78624993de4f3398/.tmp/colfamily11/f70a6fd3e50a43faa0ec678d364a1820 2024-11-13T16:11:28,439 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/72d273770314688e78624993de4f3398/.tmp/colfamily11/f70a6fd3e50a43faa0ec678d364a1820 as hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/72d273770314688e78624993de4f3398/colfamily11/f70a6fd3e50a43faa0ec678d364a1820 2024-11-13T16:11:28,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741893_1069 (size=39263) 2024-11-13T16:11:28,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741893_1069 (size=39263) 2024-11-13T16:11:28,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741893_1069 (size=39263) 2024-11-13T16:11:28,447 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/97998e7179d85b23a3f5928cd6511629/.tmp/colfamily11/52e218e5977c4530941e019a4d0569b7 2024-11-13T16:11:28,448 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/72d273770314688e78624993de4f3398/colfamily11/f70a6fd3e50a43faa0ec678d364a1820, entries=1, sequenceid=5, filesize=4.7 K 2024-11-13T16:11:28,449 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~48 B/48, heapSize ~360 B/360, currentSize=0 B/0 for 72d273770314688e78624993de4f3398 in 56ms, sequenceid=5, compaction requested=false 2024-11-13T16:11:28,454 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/72d273770314688e78624993de4f3398/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T16:11:28,454 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/97998e7179d85b23a3f5928cd6511629/.tmp/colfamily11/52e218e5977c4530941e019a4d0569b7 as hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/97998e7179d85b23a3f5928cd6511629/colfamily11/52e218e5977c4530941e019a4d0569b7 2024-11-13T16:11:28,455 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731514091776.72d273770314688e78624993de4f3398. 2024-11-13T16:11:28,455 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 72d273770314688e78624993de4f3398: Waiting for close lock at 1731514288392Running coprocessor pre-close hooks at 1731514288392Disabling compacts and flushes for region at 1731514288392Disabling writes for close at 1731514288393 (+1 ms)Obtaining lock to block concurrent updates at 1731514288393Preparing flush snapshotting stores in 72d273770314688e78624993de4f3398 at 1731514288393Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731514091776.72d273770314688e78624993de4f3398., syncing WAL and waiting on mvcc, flushsize=dataSize=48, getHeapSize=840, getOffHeapSize=0, getCellsCount=1 at 1731514288393Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731514091776.72d273770314688e78624993de4f3398. at 1731514288393Flushing 72d273770314688e78624993de4f3398/colfamily11: creating writer at 1731514288393Flushing 72d273770314688e78624993de4f3398/colfamily11: appending metadata at 1731514288411 (+18 ms)Flushing 72d273770314688e78624993de4f3398/colfamily11: closing flushed file at 1731514288411Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36793490: reopening flushed file at 1731514288438 (+27 ms)Finished flush of dataSize ~48 B/48, heapSize ~360 B/360, currentSize=0 B/0 for 72d273770314688e78624993de4f3398 in 56ms, sequenceid=5, compaction requested=false at 1731514288449 (+11 ms)Writing region close event to WAL at 1731514288450 (+1 ms)Running coprocessor post-close hooks at 1731514288454 (+4 ms)Closed at 1731514288454 2024-11-13T16:11:28,455 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731514091776.72d273770314688e78624993de4f3398. 2024-11-13T16:11:28,455 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3945ce860c8642051e17517944a83136, disabling compactions & flushes 2024-11-13T16:11:28,455 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731514091776.3945ce860c8642051e17517944a83136. 2024-11-13T16:11:28,455 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731514091776.3945ce860c8642051e17517944a83136. 2024-11-13T16:11:28,455 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731514091776.3945ce860c8642051e17517944a83136. after waiting 0 ms 2024-11-13T16:11:28,455 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731514091776.3945ce860c8642051e17517944a83136. 2024-11-13T16:11:28,462 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/97998e7179d85b23a3f5928cd6511629/colfamily11/52e218e5977c4530941e019a4d0569b7, entries=703, sequenceid=5, filesize=38.3 K 2024-11-13T16:11:28,463 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 97998e7179d85b23a3f5928cd6511629 in 64ms, sequenceid=5, compaction requested=false 2024-11-13T16:11:28,468 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3945ce860c8642051e17517944a83136/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-13T16:11:28,469 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731514091776.3945ce860c8642051e17517944a83136. 2024-11-13T16:11:28,469 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3945ce860c8642051e17517944a83136: Waiting for close lock at 1731514288455Running coprocessor pre-close hooks at 1731514288455Disabling compacts and flushes for region at 1731514288455Disabling writes for close at 1731514288455Writing region close event to WAL at 1731514288459 (+4 ms)Running coprocessor post-close hooks at 1731514288469 (+10 ms)Closed at 1731514288469 2024-11-13T16:11:28,469 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731514091776.3945ce860c8642051e17517944a83136. 2024-11-13T16:11:28,469 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6be98d350311c17b11fdb9e9a1238fca, disabling compactions & flushes 2024-11-13T16:11:28,470 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731514091776.6be98d350311c17b11fdb9e9a1238fca. 2024-11-13T16:11:28,470 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731514091776.6be98d350311c17b11fdb9e9a1238fca. 2024-11-13T16:11:28,470 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731514091776.6be98d350311c17b11fdb9e9a1238fca. after waiting 0 ms 2024-11-13T16:11:28,470 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731514091776.6be98d350311c17b11fdb9e9a1238fca. 2024-11-13T16:11:28,470 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/97998e7179d85b23a3f5928cd6511629/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T16:11:28,471 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731514091776.97998e7179d85b23a3f5928cd6511629. 2024-11-13T16:11:28,471 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 97998e7179d85b23a3f5928cd6511629: Waiting for close lock at 1731514288399Running coprocessor pre-close hooks at 1731514288399Disabling compacts and flushes for region at 1731514288399Disabling writes for close at 1731514288399Obtaining lock to block concurrent updates at 1731514288399Preparing flush snapshotting stores in 97998e7179d85b23a3f5928cd6511629 at 1731514288399Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731514091776.97998e7179d85b23a3f5928cd6511629., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731514288400 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731514091776.97998e7179d85b23a3f5928cd6511629. at 1731514288400Flushing 97998e7179d85b23a3f5928cd6511629/colfamily11: creating writer at 1731514288400Flushing 97998e7179d85b23a3f5928cd6511629/colfamily11: appending metadata at 1731514288425 (+25 ms)Flushing 97998e7179d85b23a3f5928cd6511629/colfamily11: closing flushed file at 1731514288425Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7107c6bc: reopening flushed file at 1731514288453 (+28 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 97998e7179d85b23a3f5928cd6511629 in 64ms, sequenceid=5, compaction requested=false at 1731514288463 (+10 ms)Writing region close event to WAL at 1731514288464 (+1 ms)Running coprocessor post-close hooks at 1731514288471 (+7 ms)Closed at 1731514288471 2024-11-13T16:11:28,472 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731514091776.97998e7179d85b23a3f5928cd6511629. 2024-11-13T16:11:28,472 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3647edbd8aef23dda3887b61277da1d3, disabling compactions & flushes 2024-11-13T16:11:28,472 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731514091776.3647edbd8aef23dda3887b61277da1d3. 2024-11-13T16:11:28,472 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731514091776.3647edbd8aef23dda3887b61277da1d3. 2024-11-13T16:11:28,472 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731514091776.3647edbd8aef23dda3887b61277da1d3. after waiting 0 ms 2024-11-13T16:11:28,472 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731514091776.3647edbd8aef23dda3887b61277da1d3. 2024-11-13T16:11:28,472 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 3647edbd8aef23dda3887b61277da1d3 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T16:11:28,492 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/6be98d350311c17b11fdb9e9a1238fca/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-13T16:11:28,493 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731514091776.6be98d350311c17b11fdb9e9a1238fca. 2024-11-13T16:11:28,493 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6be98d350311c17b11fdb9e9a1238fca: Waiting for close lock at 1731514288469Running coprocessor pre-close hooks at 1731514288469Disabling compacts and flushes for region at 1731514288469Disabling writes for close at 1731514288470 (+1 ms)Writing region close event to WAL at 1731514288473 (+3 ms)Running coprocessor post-close hooks at 1731514288493 (+20 ms)Closed at 1731514288493 2024-11-13T16:11:28,493 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731514091776.6be98d350311c17b11fdb9e9a1238fca. 2024-11-13T16:11:28,494 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4aaf5667f132c289386d1b3b0c0ac57f, disabling compactions & flushes 2024-11-13T16:11:28,494 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731514091776.4aaf5667f132c289386d1b3b0c0ac57f. 2024-11-13T16:11:28,494 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731514091776.4aaf5667f132c289386d1b3b0c0ac57f. 2024-11-13T16:11:28,494 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731514091776.4aaf5667f132c289386d1b3b0c0ac57f. after waiting 0 ms 2024-11-13T16:11:28,494 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731514091776.4aaf5667f132c289386d1b3b0c0ac57f. 2024-11-13T16:11:28,494 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 4aaf5667f132c289386d1b3b0c0ac57f 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T16:11:28,509 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3647edbd8aef23dda3887b61277da1d3/.tmp/colfamily11/e1a9133989c54f098373ce3445fbbf6d is 52, key is nnn/colfamily11:colfamily11/1731514098739/Put/seqid=0 2024-11-13T16:11:28,518 DEBUG [RS:1;f8f85b0bb1d8:38187 {}] regionserver.HRegionServer(1351): Waiting on 1a610c2403a0b491d368473a6e1fb049, 29df624bbcb878edce7fcdf0abceea88, 34f3d3b6a6dae1f4077567bed0bd8e75, 3647edbd8aef23dda3887b61277da1d3, 519ece296a21b5ddccc38a7c928780bb, 8ba796d80fc3b65f0948c95ccbdad39d, 93f8ed38b68bd013eb0cf0e7615013ae, c4a87adf770b37d40ab6691e377ad1b5, e8725a794e6d251e1bf6e3502baa7700 2024-11-13T16:11:28,525 DEBUG [RS:2;f8f85b0bb1d8:41333 {}] regionserver.HRegionServer(1351): Waiting on 05d65f4a6bfecb3d530d15832c218252, 1588230740, 1fcf78578ccab75f1705f3c2d7bcdb14, 4aaf5667f132c289386d1b3b0c0ac57f, aa124b802c708ca1f715cb1f8c9c4eaa, e72e57b96ff9bc0bbd862ce564d79c42, f8f8b8e4ae8470fd8a1d2bf0684947ef 2024-11-13T16:11:28,529 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4aaf5667f132c289386d1b3b0c0ac57f/.tmp/colfamily11/aa03e3a537024431a8d1a2e9d59ae479 is 52, key is aaa/colfamily11:colfamily11/1731514098215/Put/seqid=0 2024-11-13T16:11:28,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741894_1070 (size=39263) 2024-11-13T16:11:28,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741894_1070 (size=39263) 2024-11-13T16:11:28,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741894_1070 (size=39263) 2024-11-13T16:11:28,567 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3647edbd8aef23dda3887b61277da1d3/.tmp/colfamily11/e1a9133989c54f098373ce3445fbbf6d 2024-11-13T16:11:28,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741895_1071 (size=39263) 2024-11-13T16:11:28,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741895_1071 (size=39263) 2024-11-13T16:11:28,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741895_1071 (size=39263) 2024-11-13T16:11:28,584 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4aaf5667f132c289386d1b3b0c0ac57f/.tmp/colfamily11/aa03e3a537024431a8d1a2e9d59ae479 2024-11-13T16:11:28,585 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3647edbd8aef23dda3887b61277da1d3/.tmp/colfamily11/e1a9133989c54f098373ce3445fbbf6d as hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3647edbd8aef23dda3887b61277da1d3/colfamily11/e1a9133989c54f098373ce3445fbbf6d 2024-11-13T16:11:28,603 INFO [RS_CLOSE_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.91 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/hbase/meta/1588230740/.tmp/info/92dd2a68fc2145db8509bd5d31152bcb 2024-11-13T16:11:28,603 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4aaf5667f132c289386d1b3b0c0ac57f/.tmp/colfamily11/aa03e3a537024431a8d1a2e9d59ae479 as hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4aaf5667f132c289386d1b3b0c0ac57f/colfamily11/aa03e3a537024431a8d1a2e9d59ae479 2024-11-13T16:11:28,604 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3647edbd8aef23dda3887b61277da1d3/colfamily11/e1a9133989c54f098373ce3445fbbf6d, entries=703, sequenceid=5, filesize=38.3 K 2024-11-13T16:11:28,610 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 3647edbd8aef23dda3887b61277da1d3 in 137ms, sequenceid=5, compaction requested=false 2024-11-13T16:11:28,624 DEBUG [RS_CLOSE_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/hbase/meta/1588230740/.tmp/info/92dd2a68fc2145db8509bd5d31152bcb as hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/hbase/meta/1588230740/info/92dd2a68fc2145db8509bd5d31152bcb 2024-11-13T16:11:28,626 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4aaf5667f132c289386d1b3b0c0ac57f/colfamily11/aa03e3a537024431a8d1a2e9d59ae479, entries=703, sequenceid=5, filesize=38.3 K 2024-11-13T16:11:28,629 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 4aaf5667f132c289386d1b3b0c0ac57f in 134ms, sequenceid=5, compaction requested=false 2024-11-13T16:11:28,632 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3647edbd8aef23dda3887b61277da1d3/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T16:11:28,638 INFO [RS_CLOSE_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/hbase/meta/1588230740/info/92dd2a68fc2145db8509bd5d31152bcb, entries=64, sequenceid=79, filesize=17.9 K 2024-11-13T16:11:28,638 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731514091776.3647edbd8aef23dda3887b61277da1d3. 2024-11-13T16:11:28,639 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3647edbd8aef23dda3887b61277da1d3: Waiting for close lock at 1731514288472Running coprocessor pre-close hooks at 1731514288472Disabling compacts and flushes for region at 1731514288472Disabling writes for close at 1731514288472Obtaining lock to block concurrent updates at 1731514288472Preparing flush snapshotting stores in 3647edbd8aef23dda3887b61277da1d3 at 1731514288472Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731514091776.3647edbd8aef23dda3887b61277da1d3., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731514288473 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731514091776.3647edbd8aef23dda3887b61277da1d3. at 1731514288474 (+1 ms)Flushing 3647edbd8aef23dda3887b61277da1d3/colfamily11: creating writer at 1731514288474Flushing 3647edbd8aef23dda3887b61277da1d3/colfamily11: appending metadata at 1731514288508 (+34 ms)Flushing 3647edbd8aef23dda3887b61277da1d3/colfamily11: closing flushed file at 1731514288508Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@195f351f: reopening flushed file at 1731514288584 (+76 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 3647edbd8aef23dda3887b61277da1d3 in 137ms, sequenceid=5, compaction requested=false at 1731514288610 (+26 ms)Writing region close event to WAL at 1731514288623 (+13 ms)Running coprocessor post-close hooks at 1731514288638 (+15 ms)Closed at 1731514288638 2024-11-13T16:11:28,639 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731514091776.3647edbd8aef23dda3887b61277da1d3. 2024-11-13T16:11:28,639 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 34f3d3b6a6dae1f4077567bed0bd8e75, disabling compactions & flushes 2024-11-13T16:11:28,639 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731514091776.34f3d3b6a6dae1f4077567bed0bd8e75. 2024-11-13T16:11:28,639 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731514091776.34f3d3b6a6dae1f4077567bed0bd8e75. 2024-11-13T16:11:28,640 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731514091776.34f3d3b6a6dae1f4077567bed0bd8e75. after waiting 0 ms 2024-11-13T16:11:28,640 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731514091776.34f3d3b6a6dae1f4077567bed0bd8e75. 2024-11-13T16:11:28,640 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 34f3d3b6a6dae1f4077567bed0bd8e75 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T16:11:28,640 INFO [RS_CLOSE_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~11.91 KB/12192, heapSize ~16.86 KB/17264, currentSize=0 B/0 for 1588230740 in 513ms, sequenceid=79, compaction requested=false 2024-11-13T16:11:28,679 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/34f3d3b6a6dae1f4077567bed0bd8e75/.tmp/colfamily11/9b63ee6d5db14d958f6cdb38d6f8dc9c is 52, key is eee/colfamily11:colfamily11/1731514098583/Put/seqid=0 2024-11-13T16:11:28,684 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4aaf5667f132c289386d1b3b0c0ac57f/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T16:11:28,685 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731514091776.4aaf5667f132c289386d1b3b0c0ac57f. 2024-11-13T16:11:28,685 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4aaf5667f132c289386d1b3b0c0ac57f: Waiting for close lock at 1731514288494Running coprocessor pre-close hooks at 1731514288494Disabling compacts and flushes for region at 1731514288494Disabling writes for close at 1731514288494Obtaining lock to block concurrent updates at 1731514288494Preparing flush snapshotting stores in 4aaf5667f132c289386d1b3b0c0ac57f at 1731514288494Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731514091776.4aaf5667f132c289386d1b3b0c0ac57f., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731514288494Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731514091776.4aaf5667f132c289386d1b3b0c0ac57f. at 1731514288499 (+5 ms)Flushing 4aaf5667f132c289386d1b3b0c0ac57f/colfamily11: creating writer at 1731514288499Flushing 4aaf5667f132c289386d1b3b0c0ac57f/colfamily11: appending metadata at 1731514288528 (+29 ms)Flushing 4aaf5667f132c289386d1b3b0c0ac57f/colfamily11: closing flushed file at 1731514288528Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16199145: reopening flushed file at 1731514288596 (+68 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 4aaf5667f132c289386d1b3b0c0ac57f in 134ms, sequenceid=5, compaction requested=false at 1731514288629 (+33 ms)Writing region close event to WAL at 1731514288644 (+15 ms)Running coprocessor post-close hooks at 1731514288685 (+41 ms)Closed at 1731514288685 2024-11-13T16:11:28,685 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731514091776.4aaf5667f132c289386d1b3b0c0ac57f. 2024-11-13T16:11:28,685 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing aa124b802c708ca1f715cb1f8c9c4eaa, disabling compactions & flushes 2024-11-13T16:11:28,685 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731514091776.aa124b802c708ca1f715cb1f8c9c4eaa. 2024-11-13T16:11:28,685 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731514091776.aa124b802c708ca1f715cb1f8c9c4eaa. 2024-11-13T16:11:28,685 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731514091776.aa124b802c708ca1f715cb1f8c9c4eaa. after waiting 0 ms 2024-11-13T16:11:28,686 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731514091776.aa124b802c708ca1f715cb1f8c9c4eaa. 2024-11-13T16:11:28,690 INFO [regionserver/f8f85b0bb1d8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-13T16:11:28,690 INFO [regionserver/f8f85b0bb1d8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-13T16:11:28,696 DEBUG [RS_CLOSE_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/hbase/meta/1588230740/recovered.edits/82.seqid, newMaxSeqId=82, maxSeqId=60 2024-11-13T16:11:28,697 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aa124b802c708ca1f715cb1f8c9c4eaa/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-13T16:11:28,697 DEBUG [RS_CLOSE_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T16:11:28,697 INFO [RS_CLOSE_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T16:11:28,697 DEBUG [RS_CLOSE_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731514288127Running coprocessor pre-close hooks at 1731514288127Disabling compacts and flushes for region at 1731514288127Disabling writes for close at 1731514288127Obtaining lock to block concurrent updates at 1731514288128 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731514288128Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=12192, getHeapSize=17984, getOffHeapSize=0, getCellsCount=64 at 1731514288128Flushing stores of hbase:meta,,1.1588230740 at 1731514288138 (+10 ms)Flushing 1588230740/info: creating writer at 1731514288139 (+1 ms)Flushing 1588230740/info: appending metadata at 1731514288149 (+10 ms)Flushing 1588230740/info: closing flushed file at 1731514288149Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4576b27: reopening flushed file at 1731514288619 (+470 ms)Finished flush of dataSize ~11.91 KB/12192, heapSize ~16.86 KB/17264, currentSize=0 B/0 for 1588230740 in 513ms, sequenceid=79, compaction requested=false at 1731514288640 (+21 ms)Writing region close event to WAL at 1731514288687 (+47 ms)Running coprocessor post-close hooks at 1731514288697 (+10 ms)Closed at 1731514288697 2024-11-13T16:11:28,698 DEBUG [RS_CLOSE_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-13T16:11:28,698 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731514091776.aa124b802c708ca1f715cb1f8c9c4eaa. 2024-11-13T16:11:28,698 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for aa124b802c708ca1f715cb1f8c9c4eaa: Waiting for close lock at 1731514288685Running coprocessor pre-close hooks at 1731514288685Disabling compacts and flushes for region at 1731514288685Disabling writes for close at 1731514288685Writing region close event to WAL at 1731514288687 (+2 ms)Running coprocessor post-close hooks at 1731514288698 (+11 ms)Closed at 1731514288698 2024-11-13T16:11:28,698 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731514091776.aa124b802c708ca1f715cb1f8c9c4eaa. 2024-11-13T16:11:28,698 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1fcf78578ccab75f1705f3c2d7bcdb14, disabling compactions & flushes 2024-11-13T16:11:28,698 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731514091776.1fcf78578ccab75f1705f3c2d7bcdb14. 2024-11-13T16:11:28,698 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731514091776.1fcf78578ccab75f1705f3c2d7bcdb14. 2024-11-13T16:11:28,698 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731514091776.1fcf78578ccab75f1705f3c2d7bcdb14. after waiting 0 ms 2024-11-13T16:11:28,698 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731514091776.1fcf78578ccab75f1705f3c2d7bcdb14. 2024-11-13T16:11:28,699 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 1fcf78578ccab75f1705f3c2d7bcdb14 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T16:11:28,705 INFO [regionserver/f8f85b0bb1d8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-13T16:11:28,705 INFO [regionserver/f8f85b0bb1d8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-13T16:11:28,719 DEBUG [RS:1;f8f85b0bb1d8:38187 {}] regionserver.HRegionServer(1351): Waiting on 1a610c2403a0b491d368473a6e1fb049, 29df624bbcb878edce7fcdf0abceea88, 34f3d3b6a6dae1f4077567bed0bd8e75, 519ece296a21b5ddccc38a7c928780bb, 8ba796d80fc3b65f0948c95ccbdad39d, 93f8ed38b68bd013eb0cf0e7615013ae, c4a87adf770b37d40ab6691e377ad1b5, e8725a794e6d251e1bf6e3502baa7700 2024-11-13T16:11:28,725 DEBUG [RS:2;f8f85b0bb1d8:41333 {}] regionserver.HRegionServer(1351): Waiting on 05d65f4a6bfecb3d530d15832c218252, 1fcf78578ccab75f1705f3c2d7bcdb14, e72e57b96ff9bc0bbd862ce564d79c42, f8f8b8e4ae8470fd8a1d2bf0684947ef 2024-11-13T16:11:28,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741896_1072 (size=39263) 2024-11-13T16:11:28,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741896_1072 (size=39263) 2024-11-13T16:11:28,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741896_1072 (size=39263) 2024-11-13T16:11:28,743 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/34f3d3b6a6dae1f4077567bed0bd8e75/.tmp/colfamily11/9b63ee6d5db14d958f6cdb38d6f8dc9c 2024-11-13T16:11:28,747 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1fcf78578ccab75f1705f3c2d7bcdb14/.tmp/colfamily11/40cc215b37b345d698044c656d4e90c1 is 52, key is ggg/colfamily11:colfamily11/1731514098343/Put/seqid=0 2024-11-13T16:11:28,773 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/34f3d3b6a6dae1f4077567bed0bd8e75/.tmp/colfamily11/9b63ee6d5db14d958f6cdb38d6f8dc9c as hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/34f3d3b6a6dae1f4077567bed0bd8e75/colfamily11/9b63ee6d5db14d958f6cdb38d6f8dc9c 2024-11-13T16:11:28,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741897_1073 (size=39263) 2024-11-13T16:11:28,788 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1fcf78578ccab75f1705f3c2d7bcdb14/.tmp/colfamily11/40cc215b37b345d698044c656d4e90c1 2024-11-13T16:11:28,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741897_1073 (size=39263) 2024-11-13T16:11:28,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741897_1073 (size=39263) 2024-11-13T16:11:28,788 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/34f3d3b6a6dae1f4077567bed0bd8e75/colfamily11/9b63ee6d5db14d958f6cdb38d6f8dc9c, entries=703, sequenceid=5, filesize=38.3 K 2024-11-13T16:11:28,792 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 34f3d3b6a6dae1f4077567bed0bd8e75 in 151ms, sequenceid=5, compaction requested=false 2024-11-13T16:11:28,811 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1fcf78578ccab75f1705f3c2d7bcdb14/.tmp/colfamily11/40cc215b37b345d698044c656d4e90c1 as hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1fcf78578ccab75f1705f3c2d7bcdb14/colfamily11/40cc215b37b345d698044c656d4e90c1 2024-11-13T16:11:28,823 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/34f3d3b6a6dae1f4077567bed0bd8e75/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T16:11:28,825 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731514091776.34f3d3b6a6dae1f4077567bed0bd8e75. 2024-11-13T16:11:28,825 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 34f3d3b6a6dae1f4077567bed0bd8e75: Waiting for close lock at 1731514288639Running coprocessor pre-close hooks at 1731514288639Disabling compacts and flushes for region at 1731514288639Disabling writes for close at 1731514288640 (+1 ms)Obtaining lock to block concurrent updates at 1731514288640Preparing flush snapshotting stores in 34f3d3b6a6dae1f4077567bed0bd8e75 at 1731514288640Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731514091776.34f3d3b6a6dae1f4077567bed0bd8e75., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731514288640Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731514091776.34f3d3b6a6dae1f4077567bed0bd8e75. at 1731514288641 (+1 ms)Flushing 34f3d3b6a6dae1f4077567bed0bd8e75/colfamily11: creating writer at 1731514288641Flushing 34f3d3b6a6dae1f4077567bed0bd8e75/colfamily11: appending metadata at 1731514288679 (+38 ms)Flushing 34f3d3b6a6dae1f4077567bed0bd8e75/colfamily11: closing flushed file at 1731514288679Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d248408: reopening flushed file at 1731514288772 (+93 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 34f3d3b6a6dae1f4077567bed0bd8e75 in 151ms, sequenceid=5, compaction requested=false at 1731514288792 (+20 ms)Writing region close event to WAL at 1731514288793 (+1 ms)Running coprocessor post-close hooks at 1731514288825 (+32 ms)Closed at 1731514288825 2024-11-13T16:11:28,826 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731514091776.34f3d3b6a6dae1f4077567bed0bd8e75. 2024-11-13T16:11:28,826 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 93f8ed38b68bd013eb0cf0e7615013ae, disabling compactions & flushes 2024-11-13T16:11:28,826 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731514091776.93f8ed38b68bd013eb0cf0e7615013ae. 2024-11-13T16:11:28,826 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731514091776.93f8ed38b68bd013eb0cf0e7615013ae. 2024-11-13T16:11:28,826 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731514091776.93f8ed38b68bd013eb0cf0e7615013ae. after waiting 0 ms 2024-11-13T16:11:28,826 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731514091776.93f8ed38b68bd013eb0cf0e7615013ae. 2024-11-13T16:11:28,826 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 93f8ed38b68bd013eb0cf0e7615013ae 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T16:11:28,830 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1fcf78578ccab75f1705f3c2d7bcdb14/colfamily11/40cc215b37b345d698044c656d4e90c1, entries=703, sequenceid=5, filesize=38.3 K 2024-11-13T16:11:28,831 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 1fcf78578ccab75f1705f3c2d7bcdb14 in 133ms, sequenceid=5, compaction requested=false 2024-11-13T16:11:28,843 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1fcf78578ccab75f1705f3c2d7bcdb14/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T16:11:28,844 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731514091776.1fcf78578ccab75f1705f3c2d7bcdb14. 2024-11-13T16:11:28,844 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1fcf78578ccab75f1705f3c2d7bcdb14: Waiting for close lock at 1731514288698Running coprocessor pre-close hooks at 1731514288698Disabling compacts and flushes for region at 1731514288698Disabling writes for close at 1731514288698Obtaining lock to block concurrent updates at 1731514288699 (+1 ms)Preparing flush snapshotting stores in 1fcf78578ccab75f1705f3c2d7bcdb14 at 1731514288699Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731514091776.1fcf78578ccab75f1705f3c2d7bcdb14., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731514288699Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731514091776.1fcf78578ccab75f1705f3c2d7bcdb14. at 1731514288700 (+1 ms)Flushing 1fcf78578ccab75f1705f3c2d7bcdb14/colfamily11: creating writer at 1731514288700Flushing 1fcf78578ccab75f1705f3c2d7bcdb14/colfamily11: appending metadata at 1731514288743 (+43 ms)Flushing 1fcf78578ccab75f1705f3c2d7bcdb14/colfamily11: closing flushed file at 1731514288743Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@424bf8c3: reopening flushed file at 1731514288807 (+64 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 1fcf78578ccab75f1705f3c2d7bcdb14 in 133ms, sequenceid=5, compaction requested=false at 1731514288831 (+24 ms)Writing region close event to WAL at 1731514288832 (+1 ms)Running coprocessor post-close hooks at 1731514288844 (+12 ms)Closed at 1731514288844 2024-11-13T16:11:28,844 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731514091776.1fcf78578ccab75f1705f3c2d7bcdb14. 2024-11-13T16:11:28,844 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 05d65f4a6bfecb3d530d15832c218252, disabling compactions & flushes 2024-11-13T16:11:28,844 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731514091776.05d65f4a6bfecb3d530d15832c218252. 2024-11-13T16:11:28,844 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731514091776.05d65f4a6bfecb3d530d15832c218252. 2024-11-13T16:11:28,845 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731514091776.05d65f4a6bfecb3d530d15832c218252. after waiting 0 ms 2024-11-13T16:11:28,845 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731514091776.05d65f4a6bfecb3d530d15832c218252. 2024-11-13T16:11:28,845 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 05d65f4a6bfecb3d530d15832c218252 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T16:11:28,855 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/93f8ed38b68bd013eb0cf0e7615013ae/.tmp/colfamily11/460ed0699f534602b541dc77d6f49e2e is 52, key is ccc/colfamily11:colfamily11/1731514098501/Put/seqid=0 2024-11-13T16:11:28,898 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/05d65f4a6bfecb3d530d15832c218252/.tmp/colfamily11/dcd7e6225a794e5f98db789c4f9f5ee4 is 52, key is kkk/colfamily11:colfamily11/1731514098491/Put/seqid=0 2024-11-13T16:11:28,919 DEBUG [RS:1;f8f85b0bb1d8:38187 {}] regionserver.HRegionServer(1351): Waiting on 1a610c2403a0b491d368473a6e1fb049, 29df624bbcb878edce7fcdf0abceea88, 519ece296a21b5ddccc38a7c928780bb, 8ba796d80fc3b65f0948c95ccbdad39d, 93f8ed38b68bd013eb0cf0e7615013ae, c4a87adf770b37d40ab6691e377ad1b5, e8725a794e6d251e1bf6e3502baa7700 2024-11-13T16:11:28,925 DEBUG [RS:2;f8f85b0bb1d8:41333 {}] regionserver.HRegionServer(1351): Waiting on 05d65f4a6bfecb3d530d15832c218252, e72e57b96ff9bc0bbd862ce564d79c42, f8f8b8e4ae8470fd8a1d2bf0684947ef 2024-11-13T16:11:28,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741898_1074 (size=39263) 2024-11-13T16:11:28,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741898_1074 (size=39263) 2024-11-13T16:11:28,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741898_1074 (size=39263) 2024-11-13T16:11:28,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741899_1075 (size=39263) 2024-11-13T16:11:28,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741899_1075 (size=39263) 2024-11-13T16:11:28,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741899_1075 (size=39263) 2024-11-13T16:11:28,932 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/93f8ed38b68bd013eb0cf0e7615013ae/.tmp/colfamily11/460ed0699f534602b541dc77d6f49e2e 2024-11-13T16:11:28,937 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/05d65f4a6bfecb3d530d15832c218252/.tmp/colfamily11/dcd7e6225a794e5f98db789c4f9f5ee4 2024-11-13T16:11:28,953 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/93f8ed38b68bd013eb0cf0e7615013ae/.tmp/colfamily11/460ed0699f534602b541dc77d6f49e2e as hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/93f8ed38b68bd013eb0cf0e7615013ae/colfamily11/460ed0699f534602b541dc77d6f49e2e 2024-11-13T16:11:28,956 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/05d65f4a6bfecb3d530d15832c218252/.tmp/colfamily11/dcd7e6225a794e5f98db789c4f9f5ee4 as hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/05d65f4a6bfecb3d530d15832c218252/colfamily11/dcd7e6225a794e5f98db789c4f9f5ee4 2024-11-13T16:11:28,968 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/93f8ed38b68bd013eb0cf0e7615013ae/colfamily11/460ed0699f534602b541dc77d6f49e2e, entries=703, sequenceid=5, filesize=38.3 K 2024-11-13T16:11:28,970 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 93f8ed38b68bd013eb0cf0e7615013ae in 144ms, sequenceid=5, compaction requested=false 2024-11-13T16:11:28,974 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/05d65f4a6bfecb3d530d15832c218252/colfamily11/dcd7e6225a794e5f98db789c4f9f5ee4, entries=703, sequenceid=5, filesize=38.3 K 2024-11-13T16:11:28,977 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 05d65f4a6bfecb3d530d15832c218252 in 132ms, sequenceid=5, compaction requested=false 2024-11-13T16:11:29,011 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/93f8ed38b68bd013eb0cf0e7615013ae/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T16:11:29,015 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731514091776.93f8ed38b68bd013eb0cf0e7615013ae. 2024-11-13T16:11:29,015 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 93f8ed38b68bd013eb0cf0e7615013ae: Waiting for close lock at 1731514288826Running coprocessor pre-close hooks at 1731514288826Disabling compacts and flushes for region at 1731514288826Disabling writes for close at 1731514288826Obtaining lock to block concurrent updates at 1731514288826Preparing flush snapshotting stores in 93f8ed38b68bd013eb0cf0e7615013ae at 1731514288826Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731514091776.93f8ed38b68bd013eb0cf0e7615013ae., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731514288827 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731514091776.93f8ed38b68bd013eb0cf0e7615013ae. at 1731514288831 (+4 ms)Flushing 93f8ed38b68bd013eb0cf0e7615013ae/colfamily11: creating writer at 1731514288831Flushing 93f8ed38b68bd013eb0cf0e7615013ae/colfamily11: appending metadata at 1731514288854 (+23 ms)Flushing 93f8ed38b68bd013eb0cf0e7615013ae/colfamily11: closing flushed file at 1731514288854Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@233c4d6: reopening flushed file at 1731514288948 (+94 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 93f8ed38b68bd013eb0cf0e7615013ae in 144ms, sequenceid=5, compaction requested=false at 1731514288970 (+22 ms)Writing region close event to WAL at 1731514288977 (+7 ms)Running coprocessor post-close hooks at 1731514289015 (+38 ms)Closed at 1731514289015 2024-11-13T16:11:29,015 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731514091776.93f8ed38b68bd013eb0cf0e7615013ae. 2024-11-13T16:11:29,016 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 519ece296a21b5ddccc38a7c928780bb, disabling compactions & flushes 2024-11-13T16:11:29,016 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731514091776.519ece296a21b5ddccc38a7c928780bb. 2024-11-13T16:11:29,016 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731514091776.519ece296a21b5ddccc38a7c928780bb. 2024-11-13T16:11:29,016 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731514091776.519ece296a21b5ddccc38a7c928780bb. after waiting 0 ms 2024-11-13T16:11:29,016 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731514091776.519ece296a21b5ddccc38a7c928780bb. 2024-11-13T16:11:29,047 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/05d65f4a6bfecb3d530d15832c218252/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T16:11:29,048 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731514091776.05d65f4a6bfecb3d530d15832c218252. 2024-11-13T16:11:29,048 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 05d65f4a6bfecb3d530d15832c218252: Waiting for close lock at 1731514288844Running coprocessor pre-close hooks at 1731514288844Disabling compacts and flushes for region at 1731514288844Disabling writes for close at 1731514288845 (+1 ms)Obtaining lock to block concurrent updates at 1731514288845Preparing flush snapshotting stores in 05d65f4a6bfecb3d530d15832c218252 at 1731514288845Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731514091776.05d65f4a6bfecb3d530d15832c218252., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731514288845Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731514091776.05d65f4a6bfecb3d530d15832c218252. at 1731514288847 (+2 ms)Flushing 05d65f4a6bfecb3d530d15832c218252/colfamily11: creating writer at 1731514288847Flushing 05d65f4a6bfecb3d530d15832c218252/colfamily11: appending metadata at 1731514288897 (+50 ms)Flushing 05d65f4a6bfecb3d530d15832c218252/colfamily11: closing flushed file at 1731514288897Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@dc4e1fc: reopening flushed file at 1731514288954 (+57 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 05d65f4a6bfecb3d530d15832c218252 in 132ms, sequenceid=5, compaction requested=false at 1731514288977 (+23 ms)Writing region close event to WAL at 1731514289023 (+46 ms)Running coprocessor post-close hooks at 1731514289048 (+25 ms)Closed at 1731514289048 2024-11-13T16:11:29,048 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731514091776.05d65f4a6bfecb3d530d15832c218252. 2024-11-13T16:11:29,049 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e72e57b96ff9bc0bbd862ce564d79c42, disabling compactions & flushes 2024-11-13T16:11:29,049 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731514091776.e72e57b96ff9bc0bbd862ce564d79c42. 2024-11-13T16:11:29,049 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731514091776.e72e57b96ff9bc0bbd862ce564d79c42. 2024-11-13T16:11:29,049 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731514091776.e72e57b96ff9bc0bbd862ce564d79c42. after waiting 0 ms 2024-11-13T16:11:29,049 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731514091776.e72e57b96ff9bc0bbd862ce564d79c42. 2024-11-13T16:11:29,049 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing e72e57b96ff9bc0bbd862ce564d79c42 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T16:11:29,053 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/519ece296a21b5ddccc38a7c928780bb/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-13T16:11:29,055 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731514091776.519ece296a21b5ddccc38a7c928780bb. 2024-11-13T16:11:29,055 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 519ece296a21b5ddccc38a7c928780bb: Waiting for close lock at 1731514289016Running coprocessor pre-close hooks at 1731514289016Disabling compacts and flushes for region at 1731514289016Disabling writes for close at 1731514289016Writing region close event to WAL at 1731514289029 (+13 ms)Running coprocessor post-close hooks at 1731514289054 (+25 ms)Closed at 1731514289055 (+1 ms) 2024-11-13T16:11:29,055 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731514091776.519ece296a21b5ddccc38a7c928780bb. 2024-11-13T16:11:29,055 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8ba796d80fc3b65f0948c95ccbdad39d, disabling compactions & flushes 2024-11-13T16:11:29,055 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731514091776.8ba796d80fc3b65f0948c95ccbdad39d. 2024-11-13T16:11:29,055 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731514091776.8ba796d80fc3b65f0948c95ccbdad39d. 2024-11-13T16:11:29,055 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731514091776.8ba796d80fc3b65f0948c95ccbdad39d. after waiting 0 ms 2024-11-13T16:11:29,056 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731514091776.8ba796d80fc3b65f0948c95ccbdad39d. 2024-11-13T16:11:29,078 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8ba796d80fc3b65f0948c95ccbdad39d/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-13T16:11:29,079 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731514091776.8ba796d80fc3b65f0948c95ccbdad39d. 2024-11-13T16:11:29,080 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8ba796d80fc3b65f0948c95ccbdad39d: Waiting for close lock at 1731514289055Running coprocessor pre-close hooks at 1731514289055Disabling compacts and flushes for region at 1731514289055Disabling writes for close at 1731514289055Writing region close event to WAL at 1731514289060 (+5 ms)Running coprocessor post-close hooks at 1731514289079 (+19 ms)Closed at 1731514289079 2024-11-13T16:11:29,080 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e72e57b96ff9bc0bbd862ce564d79c42/.tmp/colfamily11/6c6e1053f23040b3af7372765bf01788 is 52, key is ddd/colfamily11:colfamily11/1731514098294/Put/seqid=0 2024-11-13T16:11:29,080 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731514091776.8ba796d80fc3b65f0948c95ccbdad39d. 2024-11-13T16:11:29,080 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 29df624bbcb878edce7fcdf0abceea88, disabling compactions & flushes 2024-11-13T16:11:29,080 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731514091776.29df624bbcb878edce7fcdf0abceea88. 2024-11-13T16:11:29,080 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731514091776.29df624bbcb878edce7fcdf0abceea88. 2024-11-13T16:11:29,080 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731514091776.29df624bbcb878edce7fcdf0abceea88. after waiting 0 ms 2024-11-13T16:11:29,080 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731514091776.29df624bbcb878edce7fcdf0abceea88. 2024-11-13T16:11:29,108 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/29df624bbcb878edce7fcdf0abceea88/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-13T16:11:29,109 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731514091776.29df624bbcb878edce7fcdf0abceea88. 2024-11-13T16:11:29,109 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 29df624bbcb878edce7fcdf0abceea88: Waiting for close lock at 1731514289080Running coprocessor pre-close hooks at 1731514289080Disabling compacts and flushes for region at 1731514289080Disabling writes for close at 1731514289080Writing region close event to WAL at 1731514289090 (+10 ms)Running coprocessor post-close hooks at 1731514289109 (+19 ms)Closed at 1731514289109 2024-11-13T16:11:29,109 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731514091776.29df624bbcb878edce7fcdf0abceea88. 2024-11-13T16:11:29,110 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c4a87adf770b37d40ab6691e377ad1b5, disabling compactions & flushes 2024-11-13T16:11:29,110 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731514091776.c4a87adf770b37d40ab6691e377ad1b5. 2024-11-13T16:11:29,110 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731514091776.c4a87adf770b37d40ab6691e377ad1b5. 2024-11-13T16:11:29,110 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731514091776.c4a87adf770b37d40ab6691e377ad1b5. after waiting 0 ms 2024-11-13T16:11:29,110 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731514091776.c4a87adf770b37d40ab6691e377ad1b5. 2024-11-13T16:11:29,110 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing c4a87adf770b37d40ab6691e377ad1b5 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T16:11:29,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741900_1076 (size=39263) 2024-11-13T16:11:29,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741900_1076 (size=39263) 2024-11-13T16:11:29,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741900_1076 (size=39263) 2024-11-13T16:11:29,122 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e72e57b96ff9bc0bbd862ce564d79c42/.tmp/colfamily11/6c6e1053f23040b3af7372765bf01788 2024-11-13T16:11:29,122 INFO [RS:1;f8f85b0bb1d8:38187 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close 2024-11-13T16:11:29,123 DEBUG [RS:1;f8f85b0bb1d8:38187 {}] regionserver.HRegionServer(1325): Online Regions={c4a87adf770b37d40ab6691e377ad1b5=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731514091776.c4a87adf770b37d40ab6691e377ad1b5., 1a610c2403a0b491d368473a6e1fb049=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731514091776.1a610c2403a0b491d368473a6e1fb049., e8725a794e6d251e1bf6e3502baa7700=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731514091776.e8725a794e6d251e1bf6e3502baa7700.} 2024-11-13T16:11:29,123 DEBUG [RS:1;f8f85b0bb1d8:38187 {}] regionserver.HRegionServer(1351): Waiting on 1a610c2403a0b491d368473a6e1fb049, c4a87adf770b37d40ab6691e377ad1b5, e8725a794e6d251e1bf6e3502baa7700 2024-11-13T16:11:29,125 INFO [RS:2;f8f85b0bb1d8:41333 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-13T16:11:29,126 DEBUG [RS:2;f8f85b0bb1d8:41333 {}] regionserver.HRegionServer(1325): Online Regions={e72e57b96ff9bc0bbd862ce564d79c42=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731514091776.e72e57b96ff9bc0bbd862ce564d79c42., f8f8b8e4ae8470fd8a1d2bf0684947ef=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731514091776.f8f8b8e4ae8470fd8a1d2bf0684947ef.} 2024-11-13T16:11:29,126 DEBUG [RS:2;f8f85b0bb1d8:41333 {}] regionserver.HRegionServer(1351): Waiting on e72e57b96ff9bc0bbd862ce564d79c42, f8f8b8e4ae8470fd8a1d2bf0684947ef 2024-11-13T16:11:29,136 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e72e57b96ff9bc0bbd862ce564d79c42/.tmp/colfamily11/6c6e1053f23040b3af7372765bf01788 as hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e72e57b96ff9bc0bbd862ce564d79c42/colfamily11/6c6e1053f23040b3af7372765bf01788 2024-11-13T16:11:29,137 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c4a87adf770b37d40ab6691e377ad1b5/.tmp/colfamily11/8273ead4f1174dfbb3971a8340c35df9 is 52, key is vvv/colfamily11:colfamily11/1731514098873/Put/seqid=0 2024-11-13T16:11:29,144 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e72e57b96ff9bc0bbd862ce564d79c42/colfamily11/6c6e1053f23040b3af7372765bf01788, entries=703, sequenceid=5, filesize=38.3 K 2024-11-13T16:11:29,148 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for e72e57b96ff9bc0bbd862ce564d79c42 in 99ms, sequenceid=5, compaction requested=false 2024-11-13T16:11:29,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741901_1077 (size=39263) 2024-11-13T16:11:29,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741901_1077 (size=39263) 2024-11-13T16:11:29,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741901_1077 (size=39263) 2024-11-13T16:11:29,157 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c4a87adf770b37d40ab6691e377ad1b5/.tmp/colfamily11/8273ead4f1174dfbb3971a8340c35df9 2024-11-13T16:11:29,165 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e72e57b96ff9bc0bbd862ce564d79c42/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T16:11:29,166 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731514091776.e72e57b96ff9bc0bbd862ce564d79c42. 2024-11-13T16:11:29,166 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e72e57b96ff9bc0bbd862ce564d79c42: Waiting for close lock at 1731514289048Running coprocessor pre-close hooks at 1731514289048Disabling compacts and flushes for region at 1731514289048Disabling writes for close at 1731514289049 (+1 ms)Obtaining lock to block concurrent updates at 1731514289049Preparing flush snapshotting stores in e72e57b96ff9bc0bbd862ce564d79c42 at 1731514289049Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731514091776.e72e57b96ff9bc0bbd862ce564d79c42., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731514289050 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731514091776.e72e57b96ff9bc0bbd862ce564d79c42. at 1731514289050Flushing e72e57b96ff9bc0bbd862ce564d79c42/colfamily11: creating writer at 1731514289051 (+1 ms)Flushing e72e57b96ff9bc0bbd862ce564d79c42/colfamily11: appending metadata at 1731514289079 (+28 ms)Flushing e72e57b96ff9bc0bbd862ce564d79c42/colfamily11: closing flushed file at 1731514289079Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@436b7bb6: reopening flushed file at 1731514289134 (+55 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for e72e57b96ff9bc0bbd862ce564d79c42 in 99ms, sequenceid=5, compaction requested=false at 1731514289148 (+14 ms)Writing region close event to WAL at 1731514289151 (+3 ms)Running coprocessor post-close hooks at 1731514289166 (+15 ms)Closed at 1731514289166 2024-11-13T16:11:29,167 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731514091776.e72e57b96ff9bc0bbd862ce564d79c42. 2024-11-13T16:11:29,167 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f8f8b8e4ae8470fd8a1d2bf0684947ef, disabling compactions & flushes 2024-11-13T16:11:29,167 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731514091776.f8f8b8e4ae8470fd8a1d2bf0684947ef. 2024-11-13T16:11:29,167 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731514091776.f8f8b8e4ae8470fd8a1d2bf0684947ef. 2024-11-13T16:11:29,167 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731514091776.f8f8b8e4ae8470fd8a1d2bf0684947ef. after waiting 0 ms 2024-11-13T16:11:29,167 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731514091776.f8f8b8e4ae8470fd8a1d2bf0684947ef. 2024-11-13T16:11:29,167 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing f8f8b8e4ae8470fd8a1d2bf0684947ef 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T16:11:29,169 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c4a87adf770b37d40ab6691e377ad1b5/.tmp/colfamily11/8273ead4f1174dfbb3971a8340c35df9 as hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c4a87adf770b37d40ab6691e377ad1b5/colfamily11/8273ead4f1174dfbb3971a8340c35df9 2024-11-13T16:11:29,183 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c4a87adf770b37d40ab6691e377ad1b5/colfamily11/8273ead4f1174dfbb3971a8340c35df9, entries=703, sequenceid=5, filesize=38.3 K 2024-11-13T16:11:29,185 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for c4a87adf770b37d40ab6691e377ad1b5 in 74ms, sequenceid=5, compaction requested=false 2024-11-13T16:11:29,189 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f8f8b8e4ae8470fd8a1d2bf0684947ef/.tmp/colfamily11/20a888679a944f97876f6f052d60f965 is 52, key is rrr/colfamily11:colfamily11/1731514098578/Put/seqid=0 2024-11-13T16:11:29,193 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c4a87adf770b37d40ab6691e377ad1b5/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T16:11:29,194 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731514091776.c4a87adf770b37d40ab6691e377ad1b5. 2024-11-13T16:11:29,194 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c4a87adf770b37d40ab6691e377ad1b5: Waiting for close lock at 1731514289110Running coprocessor pre-close hooks at 1731514289110Disabling compacts and flushes for region at 1731514289110Disabling writes for close at 1731514289110Obtaining lock to block concurrent updates at 1731514289110Preparing flush snapshotting stores in c4a87adf770b37d40ab6691e377ad1b5 at 1731514289110Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731514091776.c4a87adf770b37d40ab6691e377ad1b5., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731514289110Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731514091776.c4a87adf770b37d40ab6691e377ad1b5. at 1731514289112 (+2 ms)Flushing c4a87adf770b37d40ab6691e377ad1b5/colfamily11: creating writer at 1731514289112Flushing c4a87adf770b37d40ab6691e377ad1b5/colfamily11: appending metadata at 1731514289136 (+24 ms)Flushing c4a87adf770b37d40ab6691e377ad1b5/colfamily11: closing flushed file at 1731514289136Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54965a23: reopening flushed file at 1731514289163 (+27 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for c4a87adf770b37d40ab6691e377ad1b5 in 74ms, sequenceid=5, compaction requested=false at 1731514289185 (+22 ms)Writing region close event to WAL at 1731514289187 (+2 ms)Running coprocessor post-close hooks at 1731514289194 (+7 ms)Closed at 1731514289194 2024-11-13T16:11:29,194 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731514091776.c4a87adf770b37d40ab6691e377ad1b5. 2024-11-13T16:11:29,195 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1a610c2403a0b491d368473a6e1fb049, disabling compactions & flushes 2024-11-13T16:11:29,195 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731514091776.1a610c2403a0b491d368473a6e1fb049. 2024-11-13T16:11:29,195 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731514091776.1a610c2403a0b491d368473a6e1fb049. 2024-11-13T16:11:29,195 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731514091776.1a610c2403a0b491d368473a6e1fb049. after waiting 0 ms 2024-11-13T16:11:29,195 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731514091776.1a610c2403a0b491d368473a6e1fb049. 2024-11-13T16:11:29,195 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 1a610c2403a0b491d368473a6e1fb049 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T16:11:29,200 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=f8f85b0bb1d8:42949 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 10 more 2024-11-13T16:11:29,229 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1a610c2403a0b491d368473a6e1fb049/.tmp/colfamily11/6e9cf0371c6c45e0987c76c040f534aa is 52, key is hhh/colfamily11:colfamily11/1731514098651/Put/seqid=0 2024-11-13T16:11:29,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741902_1078 (size=39263) 2024-11-13T16:11:29,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741902_1078 (size=39263) 2024-11-13T16:11:29,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741902_1078 (size=39263) 2024-11-13T16:11:29,249 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f8f8b8e4ae8470fd8a1d2bf0684947ef/.tmp/colfamily11/20a888679a944f97876f6f052d60f965 2024-11-13T16:11:29,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741903_1079 (size=39263) 2024-11-13T16:11:29,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741903_1079 (size=39263) 2024-11-13T16:11:29,270 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1a610c2403a0b491d368473a6e1fb049/.tmp/colfamily11/6e9cf0371c6c45e0987c76c040f534aa 2024-11-13T16:11:29,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741903_1079 (size=39263) 2024-11-13T16:11:29,276 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f8f8b8e4ae8470fd8a1d2bf0684947ef/.tmp/colfamily11/20a888679a944f97876f6f052d60f965 as hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f8f8b8e4ae8470fd8a1d2bf0684947ef/colfamily11/20a888679a944f97876f6f052d60f965 2024-11-13T16:11:29,285 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f8f8b8e4ae8470fd8a1d2bf0684947ef/colfamily11/20a888679a944f97876f6f052d60f965, entries=703, sequenceid=5, filesize=38.3 K 2024-11-13T16:11:29,286 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for f8f8b8e4ae8470fd8a1d2bf0684947ef in 119ms, sequenceid=5, compaction requested=false 2024-11-13T16:11:29,296 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f8f8b8e4ae8470fd8a1d2bf0684947ef/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T16:11:29,297 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1a610c2403a0b491d368473a6e1fb049/.tmp/colfamily11/6e9cf0371c6c45e0987c76c040f534aa as hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1a610c2403a0b491d368473a6e1fb049/colfamily11/6e9cf0371c6c45e0987c76c040f534aa 2024-11-13T16:11:29,299 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731514091776.f8f8b8e4ae8470fd8a1d2bf0684947ef. 2024-11-13T16:11:29,299 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f8f8b8e4ae8470fd8a1d2bf0684947ef: Waiting for close lock at 1731514289167Running coprocessor pre-close hooks at 1731514289167Disabling compacts and flushes for region at 1731514289167Disabling writes for close at 1731514289167Obtaining lock to block concurrent updates at 1731514289167Preparing flush snapshotting stores in f8f8b8e4ae8470fd8a1d2bf0684947ef at 1731514289167Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731514091776.f8f8b8e4ae8470fd8a1d2bf0684947ef., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731514289168 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731514091776.f8f8b8e4ae8470fd8a1d2bf0684947ef. at 1731514289168Flushing f8f8b8e4ae8470fd8a1d2bf0684947ef/colfamily11: creating writer at 1731514289168Flushing f8f8b8e4ae8470fd8a1d2bf0684947ef/colfamily11: appending metadata at 1731514289189 (+21 ms)Flushing f8f8b8e4ae8470fd8a1d2bf0684947ef/colfamily11: closing flushed file at 1731514289189Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@237fd338: reopening flushed file at 1731514289274 (+85 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for f8f8b8e4ae8470fd8a1d2bf0684947ef in 119ms, sequenceid=5, compaction requested=false at 1731514289286 (+12 ms)Writing region close event to WAL at 1731514289287 (+1 ms)Running coprocessor post-close hooks at 1731514289299 (+12 ms)Closed at 1731514289299 2024-11-13T16:11:29,299 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731514091776.f8f8b8e4ae8470fd8a1d2bf0684947ef. 2024-11-13T16:11:29,309 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1a610c2403a0b491d368473a6e1fb049/colfamily11/6e9cf0371c6c45e0987c76c040f534aa, entries=703, sequenceid=5, filesize=38.3 K 2024-11-13T16:11:29,315 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 1a610c2403a0b491d368473a6e1fb049 in 120ms, sequenceid=5, compaction requested=false 2024-11-13T16:11:29,323 DEBUG [RS:1;f8f85b0bb1d8:38187 {}] regionserver.HRegionServer(1351): Waiting on 1a610c2403a0b491d368473a6e1fb049, e8725a794e6d251e1bf6e3502baa7700 2024-11-13T16:11:29,326 INFO [RS:2;f8f85b0bb1d8:41333 {}] regionserver.HRegionServer(976): stopping server f8f85b0bb1d8,41333,1731514088798; all regions closed. 2024-11-13T16:11:29,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741865_1041 (size=14124) 2024-11-13T16:11:29,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741865_1041 (size=14124) 2024-11-13T16:11:29,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741865_1041 (size=14124) 2024-11-13T16:11:29,334 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1a610c2403a0b491d368473a6e1fb049/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T16:11:29,339 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731514091776.1a610c2403a0b491d368473a6e1fb049. 2024-11-13T16:11:29,339 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1a610c2403a0b491d368473a6e1fb049: Waiting for close lock at 1731514289194Running coprocessor pre-close hooks at 1731514289194Disabling compacts and flushes for region at 1731514289194Disabling writes for close at 1731514289195 (+1 ms)Obtaining lock to block concurrent updates at 1731514289195Preparing flush snapshotting stores in 1a610c2403a0b491d368473a6e1fb049 at 1731514289195Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731514091776.1a610c2403a0b491d368473a6e1fb049., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731514289195Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731514091776.1a610c2403a0b491d368473a6e1fb049. at 1731514289196 (+1 ms)Flushing 1a610c2403a0b491d368473a6e1fb049/colfamily11: creating writer at 1731514289196Flushing 1a610c2403a0b491d368473a6e1fb049/colfamily11: appending metadata at 1731514289228 (+32 ms)Flushing 1a610c2403a0b491d368473a6e1fb049/colfamily11: closing flushed file at 1731514289228Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d63ac3b: reopening flushed file at 1731514289296 (+68 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 1a610c2403a0b491d368473a6e1fb049 in 120ms, sequenceid=5, compaction requested=false at 1731514289315 (+19 ms)Writing region close event to WAL at 1731514289326 (+11 ms)Running coprocessor post-close hooks at 1731514289339 (+13 ms)Closed at 1731514289339 2024-11-13T16:11:29,340 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731514091776.1a610c2403a0b491d368473a6e1fb049. 2024-11-13T16:11:29,340 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e8725a794e6d251e1bf6e3502baa7700, disabling compactions & flushes 2024-11-13T16:11:29,340 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731514091776.e8725a794e6d251e1bf6e3502baa7700. 2024-11-13T16:11:29,340 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731514091776.e8725a794e6d251e1bf6e3502baa7700. 2024-11-13T16:11:29,340 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731514091776.e8725a794e6d251e1bf6e3502baa7700. after waiting 0 ms 2024-11-13T16:11:29,340 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731514091776.e8725a794e6d251e1bf6e3502baa7700. 2024-11-13T16:11:29,348 DEBUG [RS:2;f8f85b0bb1d8:41333 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/oldWALs 2024-11-13T16:11:29,348 INFO [RS:2;f8f85b0bb1d8:41333 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f8f85b0bb1d8%2C41333%2C1731514088798.meta:.meta(num 1731514280437) 2024-11-13T16:11:29,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741833_1009 (size=316519) 2024-11-13T16:11:29,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741833_1009 (size=316519) 2024-11-13T16:11:29,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741833_1009 (size=316519) 2024-11-13T16:11:29,377 DEBUG [RS:2;f8f85b0bb1d8:41333 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/oldWALs 2024-11-13T16:11:29,377 INFO [RS:2;f8f85b0bb1d8:41333 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f8f85b0bb1d8%2C41333%2C1731514088798:(num 1731514090856) 2024-11-13T16:11:29,377 DEBUG [RS:2;f8f85b0bb1d8:41333 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T16:11:29,377 INFO [RS:2;f8f85b0bb1d8:41333 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T16:11:29,377 INFO [RS:2;f8f85b0bb1d8:41333 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T16:11:29,377 INFO [RS:2;f8f85b0bb1d8:41333 {}] hbase.ChoreService(370): Chore service for: regionserver/f8f85b0bb1d8:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-13T16:11:29,377 INFO [RS:2;f8f85b0bb1d8:41333 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T16:11:29,378 INFO [regionserver/f8f85b0bb1d8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T16:11:29,378 INFO [RS:2;f8f85b0bb1d8:41333 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:41333 2024-11-13T16:11:29,380 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e8725a794e6d251e1bf6e3502baa7700/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-13T16:11:29,383 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731514091776.e8725a794e6d251e1bf6e3502baa7700. 2024-11-13T16:11:29,383 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e8725a794e6d251e1bf6e3502baa7700: Waiting for close lock at 1731514289340Running coprocessor pre-close hooks at 1731514289340Disabling compacts and flushes for region at 1731514289340Disabling writes for close at 1731514289340Writing region close event to WAL at 1731514289363 (+23 ms)Running coprocessor post-close hooks at 1731514289383 (+20 ms)Closed at 1731514289383 2024-11-13T16:11:29,383 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731514091776.e8725a794e6d251e1bf6e3502baa7700. 2024-11-13T16:11:29,399 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41333-0x1013621fa650003, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f8f85b0bb1d8,41333,1731514088798 2024-11-13T16:11:29,399 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42949-0x1013621fa650000, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T16:11:29,399 INFO [RS:2;f8f85b0bb1d8:41333 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T16:11:29,409 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f8f85b0bb1d8,41333,1731514088798] 2024-11-13T16:11:29,417 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f8f85b0bb1d8,41333,1731514088798 already deleted, retry=false 2024-11-13T16:11:29,419 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; f8f85b0bb1d8,41333,1731514088798 expired; onlineServers=1 2024-11-13T16:11:29,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41333-0x1013621fa650003, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T16:11:29,509 INFO [RS:2;f8f85b0bb1d8:41333 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T16:11:29,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41333-0x1013621fa650003, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T16:11:29,509 INFO [RS:2;f8f85b0bb1d8:41333 {}] regionserver.HRegionServer(1031): Exiting; stopping=f8f85b0bb1d8,41333,1731514088798; zookeeper connection closed. 2024-11-13T16:11:29,510 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@34ed119b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@34ed119b 2024-11-13T16:11:29,523 INFO [RS:1;f8f85b0bb1d8:38187 {}] regionserver.HRegionServer(976): stopping server f8f85b0bb1d8,38187,1731514088725; all regions closed. 2024-11-13T16:11:29,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741835_1011 (size=353044) 2024-11-13T16:11:29,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741835_1011 (size=353044) 2024-11-13T16:11:29,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741835_1011 (size=353044) 2024-11-13T16:11:29,528 DEBUG [RS:1;f8f85b0bb1d8:38187 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/oldWALs 2024-11-13T16:11:29,528 INFO [RS:1;f8f85b0bb1d8:38187 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f8f85b0bb1d8%2C38187%2C1731514088725:(num 1731514090856) 2024-11-13T16:11:29,528 DEBUG [RS:1;f8f85b0bb1d8:38187 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T16:11:29,528 INFO [RS:1;f8f85b0bb1d8:38187 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T16:11:29,529 INFO [RS:1;f8f85b0bb1d8:38187 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T16:11:29,529 INFO [RS:1;f8f85b0bb1d8:38187 {}] hbase.ChoreService(370): Chore service for: regionserver/f8f85b0bb1d8:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-13T16:11:29,529 INFO [RS:1;f8f85b0bb1d8:38187 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T16:11:29,529 INFO [RS:1;f8f85b0bb1d8:38187 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T16:11:29,529 INFO [regionserver/f8f85b0bb1d8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T16:11:29,529 INFO [RS:1;f8f85b0bb1d8:38187 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T16:11:29,529 INFO [RS:1;f8f85b0bb1d8:38187 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T16:11:29,529 INFO [RS:1;f8f85b0bb1d8:38187 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:38187 2024-11-13T16:11:29,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42949-0x1013621fa650000, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T16:11:29,540 INFO [RS:1;f8f85b0bb1d8:38187 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T16:11:29,540 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38187-0x1013621fa650002, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f8f85b0bb1d8,38187,1731514088725 2024-11-13T16:11:29,551 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f8f85b0bb1d8,38187,1731514088725] 2024-11-13T16:11:29,559 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f8f85b0bb1d8,38187,1731514088725 already deleted, retry=false 2024-11-13T16:11:29,559 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; f8f85b0bb1d8,38187,1731514088725 expired; onlineServers=0 2024-11-13T16:11:29,559 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'f8f85b0bb1d8,42949,1731514087490' ***** 2024-11-13T16:11:29,559 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-13T16:11:29,559 INFO [M:0;f8f85b0bb1d8:42949 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T16:11:29,559 INFO [M:0;f8f85b0bb1d8:42949 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T16:11:29,559 DEBUG [M:0;f8f85b0bb1d8:42949 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-13T16:11:29,560 DEBUG [M:0;f8f85b0bb1d8:42949 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-13T16:11:29,560 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-13T16:11:29,560 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster-HFileCleaner.small.0-1731514090351 {}] cleaner.HFileCleaner(306): Exit Thread[master/f8f85b0bb1d8:0:becomeActiveMaster-HFileCleaner.small.0-1731514090351,5,FailOnTimeoutGroup] 2024-11-13T16:11:29,560 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster-HFileCleaner.large.0-1731514090338 {}] cleaner.HFileCleaner(306): Exit Thread[master/f8f85b0bb1d8:0:becomeActiveMaster-HFileCleaner.large.0-1731514090338,5,FailOnTimeoutGroup] 2024-11-13T16:11:29,560 INFO [M:0;f8f85b0bb1d8:42949 {}] hbase.ChoreService(370): Chore service for: master/f8f85b0bb1d8:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-13T16:11:29,560 INFO [M:0;f8f85b0bb1d8:42949 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T16:11:29,560 DEBUG [M:0;f8f85b0bb1d8:42949 {}] master.HMaster(1795): Stopping service threads 2024-11-13T16:11:29,560 INFO [M:0;f8f85b0bb1d8:42949 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-13T16:11:29,560 INFO [M:0;f8f85b0bb1d8:42949 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T16:11:29,560 INFO [M:0;f8f85b0bb1d8:42949 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T16:11:29,560 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-13T16:11:29,567 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42949-0x1013621fa650000, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-13T16:11:29,567 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42949-0x1013621fa650000, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:11:29,567 DEBUG [M:0;f8f85b0bb1d8:42949 {}] zookeeper.ZKUtil(347): master:42949-0x1013621fa650000, quorum=127.0.0.1:60455, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-13T16:11:29,568 WARN [M:0;f8f85b0bb1d8:42949 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-13T16:11:29,568 INFO [M:0;f8f85b0bb1d8:42949 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/.lastflushedseqids 2024-11-13T16:11:29,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741904_1080 (size=2864) 2024-11-13T16:11:29,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741904_1080 (size=2864) 2024-11-13T16:11:29,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741904_1080 (size=2864) 2024-11-13T16:11:29,583 INFO [M:0;f8f85b0bb1d8:42949 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T16:11:29,583 INFO [M:0;f8f85b0bb1d8:42949 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-13T16:11:29,583 DEBUG [M:0;f8f85b0bb1d8:42949 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T16:11:29,583 INFO [M:0;f8f85b0bb1d8:42949 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T16:11:29,583 DEBUG [M:0;f8f85b0bb1d8:42949 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T16:11:29,583 DEBUG [M:0;f8f85b0bb1d8:42949 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T16:11:29,583 DEBUG [M:0;f8f85b0bb1d8:42949 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T16:11:29,583 INFO [M:0;f8f85b0bb1d8:42949 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=336.43 KB heapSize=395.44 KB 2024-11-13T16:11:29,597 DEBUG [M:0;f8f85b0bb1d8:42949 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/bfae959af49a4556903dbd8dd16495b2 is 82, key is hbase:meta,,1/info:regioninfo/1731514281296/Put/seqid=0 2024-11-13T16:11:29,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741905_1081 (size=6071) 2024-11-13T16:11:29,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741905_1081 (size=6071) 2024-11-13T16:11:29,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741905_1081 (size=6071) 2024-11-13T16:11:29,604 INFO [M:0;f8f85b0bb1d8:42949 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=620 (bloomFilter=true), to=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/bfae959af49a4556903dbd8dd16495b2 2024-11-13T16:11:29,628 DEBUG [M:0;f8f85b0bb1d8:42949 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1bcd015967174c08a9ddb47cf4391465 is 4107, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731514094320/Put/seqid=0 2024-11-13T16:11:29,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741906_1082 (size=13328) 2024-11-13T16:11:29,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741906_1082 (size=13328) 2024-11-13T16:11:29,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741906_1082 (size=13328) 2024-11-13T16:11:29,643 INFO [M:0;f8f85b0bb1d8:42949 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=335.20 KB at sequenceid=620 (bloomFilter=true), to=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1bcd015967174c08a9ddb47cf4391465 2024-11-13T16:11:29,651 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38187-0x1013621fa650002, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T16:11:29,651 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38187-0x1013621fa650002, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T16:11:29,652 INFO [RS:1;f8f85b0bb1d8:38187 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T16:11:29,652 INFO [RS:1;f8f85b0bb1d8:38187 {}] regionserver.HRegionServer(1031): Exiting; stopping=f8f85b0bb1d8,38187,1731514088725; zookeeper connection closed. 2024-11-13T16:11:29,653 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@783a6d9e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@783a6d9e 2024-11-13T16:11:29,653 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-13T16:11:29,670 DEBUG [M:0;f8f85b0bb1d8:42949 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f7c50989824b4593ad712c81ef480539 is 69, key is f8f85b0bb1d8,38187,1731514088725/rs:state/1731514090526/Put/seqid=0 2024-11-13T16:11:29,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741907_1083 (size=5442) 2024-11-13T16:11:29,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741907_1083 (size=5442) 2024-11-13T16:11:29,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741907_1083 (size=5442) 2024-11-13T16:11:29,677 INFO [M:0;f8f85b0bb1d8:42949 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=620 (bloomFilter=true), to=hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f7c50989824b4593ad712c81ef480539 2024-11-13T16:11:29,682 INFO [M:0;f8f85b0bb1d8:42949 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f7c50989824b4593ad712c81ef480539 2024-11-13T16:11:29,683 DEBUG [M:0;f8f85b0bb1d8:42949 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/bfae959af49a4556903dbd8dd16495b2 as hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/bfae959af49a4556903dbd8dd16495b2 2024-11-13T16:11:29,688 INFO [M:0;f8f85b0bb1d8:42949 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/bfae959af49a4556903dbd8dd16495b2, entries=14, sequenceid=620, filesize=5.9 K 2024-11-13T16:11:29,689 DEBUG [M:0;f8f85b0bb1d8:42949 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1bcd015967174c08a9ddb47cf4391465 as hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1bcd015967174c08a9ddb47cf4391465 2024-11-13T16:11:29,694 INFO [M:0;f8f85b0bb1d8:42949 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1bcd015967174c08a9ddb47cf4391465, entries=79, sequenceid=620, filesize=13.0 K 2024-11-13T16:11:29,695 DEBUG [M:0;f8f85b0bb1d8:42949 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f7c50989824b4593ad712c81ef480539 as hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f7c50989824b4593ad712c81ef480539 2024-11-13T16:11:29,701 INFO [M:0;f8f85b0bb1d8:42949 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f7c50989824b4593ad712c81ef480539 2024-11-13T16:11:29,701 INFO [M:0;f8f85b0bb1d8:42949 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44161/user/jenkins/test-data/f4fdfeee-967b-1e90-27e9-52f1c17a1b46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f7c50989824b4593ad712c81ef480539, entries=3, sequenceid=620, filesize=5.3 K 2024-11-13T16:11:29,702 INFO [M:0;f8f85b0bb1d8:42949 {}] regionserver.HRegion(3140): Finished flush of dataSize ~336.43 KB/344501, heapSize ~395.14 KB/404624, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 119ms, sequenceid=620, compaction requested=false 2024-11-13T16:11:29,704 INFO [M:0;f8f85b0bb1d8:42949 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T16:11:29,704 DEBUG [M:0;f8f85b0bb1d8:42949 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731514289583Disabling compacts and flushes for region at 1731514289583Disabling writes for close at 1731514289583Obtaining lock to block concurrent updates at 1731514289583Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731514289583Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=344501, getHeapSize=404864, getOffHeapSize=0, getCellsCount=781 at 1731514289584 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731514289584Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731514289584Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731514289597 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731514289597Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731514289608 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731514289628 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731514289628Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731514289652 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731514289669 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731514289669Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c38b160: reopening flushed file at 1731514289682 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d58eba3: reopening flushed file at 1731514289688 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e6eee75: reopening flushed file at 1731514289695 (+7 ms)Finished flush of dataSize ~336.43 KB/344501, heapSize ~395.14 KB/404624, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 119ms, sequenceid=620, compaction requested=false at 1731514289702 (+7 ms)Writing region close event to WAL at 1731514289704 (+2 ms)Closed at 1731514289704 2024-11-13T16:11:29,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32863 is added to blk_1073741830_1006 (size=385864) 2024-11-13T16:11:29,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45743 is added to blk_1073741830_1006 (size=385864) 2024-11-13T16:11:29,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33087 is added to blk_1073741830_1006 (size=385864) 2024-11-13T16:11:29,710 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T16:11:29,711 INFO [M:0;f8f85b0bb1d8:42949 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-13T16:11:29,711 INFO [M:0;f8f85b0bb1d8:42949 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:42949 2024-11-13T16:11:29,711 INFO [M:0;f8f85b0bb1d8:42949 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T16:11:29,832 INFO [M:0;f8f85b0bb1d8:42949 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T16:11:29,832 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42949-0x1013621fa650000, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T16:11:29,832 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42949-0x1013621fa650000, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T16:11:29,840 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@47300910{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T16:11:29,843 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@38cb4984{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T16:11:29,843 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T16:11:29,843 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7605ef0c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T16:11:29,843 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@697bf951{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3772cfda-71de-26d3-3229-45fcb377a4d9/hadoop.log.dir/,STOPPED} 2024-11-13T16:11:29,846 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T16:11:29,846 WARN [BP-1848284120-172.17.0.3-1731514082621 heartbeating to localhost/127.0.0.1:44161 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T16:11:29,846 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T16:11:29,846 WARN [BP-1848284120-172.17.0.3-1731514082621 heartbeating to localhost/127.0.0.1:44161 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1848284120-172.17.0.3-1731514082621 (Datanode Uuid 4c2e890c-e34f-48b7-b831-a71ec0b9834d) service to localhost/127.0.0.1:44161 2024-11-13T16:11:29,847 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3772cfda-71de-26d3-3229-45fcb377a4d9/cluster_1b15728d-283e-8daa-5d9c-302860a3b764/data/data5/current/BP-1848284120-172.17.0.3-1731514082621 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T16:11:29,847 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3772cfda-71de-26d3-3229-45fcb377a4d9/cluster_1b15728d-283e-8daa-5d9c-302860a3b764/data/data6/current/BP-1848284120-172.17.0.3-1731514082621 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T16:11:29,848 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T16:11:29,851 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@60b2194c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T16:11:29,851 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7ac58d5{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T16:11:29,851 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T16:11:29,851 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@62535b13{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T16:11:29,851 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3a2ddbb3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3772cfda-71de-26d3-3229-45fcb377a4d9/hadoop.log.dir/,STOPPED} 2024-11-13T16:11:29,852 WARN [BP-1848284120-172.17.0.3-1731514082621 heartbeating to localhost/127.0.0.1:44161 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T16:11:29,852 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T16:11:29,852 WARN [BP-1848284120-172.17.0.3-1731514082621 heartbeating to localhost/127.0.0.1:44161 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1848284120-172.17.0.3-1731514082621 (Datanode Uuid fc672ff6-9eb1-4fc8-a112-e265d00093c2) service to localhost/127.0.0.1:44161 2024-11-13T16:11:29,852 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T16:11:29,853 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3772cfda-71de-26d3-3229-45fcb377a4d9/cluster_1b15728d-283e-8daa-5d9c-302860a3b764/data/data3/current/BP-1848284120-172.17.0.3-1731514082621 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T16:11:29,853 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3772cfda-71de-26d3-3229-45fcb377a4d9/cluster_1b15728d-283e-8daa-5d9c-302860a3b764/data/data4/current/BP-1848284120-172.17.0.3-1731514082621 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T16:11:29,854 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T16:11:29,859 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@75313660{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T16:11:29,860 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@10d359de{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T16:11:29,860 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T16:11:29,860 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7a83e474{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T16:11:29,860 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@31fa71ec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3772cfda-71de-26d3-3229-45fcb377a4d9/hadoop.log.dir/,STOPPED} 2024-11-13T16:11:29,861 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T16:11:29,861 WARN [BP-1848284120-172.17.0.3-1731514082621 heartbeating to localhost/127.0.0.1:44161 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T16:11:29,861 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T16:11:29,861 WARN [BP-1848284120-172.17.0.3-1731514082621 heartbeating to localhost/127.0.0.1:44161 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1848284120-172.17.0.3-1731514082621 (Datanode Uuid 3de4e64d-0c3f-4dd4-b7a5-e67576d85d5e) service to localhost/127.0.0.1:44161 2024-11-13T16:11:29,862 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3772cfda-71de-26d3-3229-45fcb377a4d9/cluster_1b15728d-283e-8daa-5d9c-302860a3b764/data/data1/current/BP-1848284120-172.17.0.3-1731514082621 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T16:11:29,863 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3772cfda-71de-26d3-3229-45fcb377a4d9/cluster_1b15728d-283e-8daa-5d9c-302860a3b764/data/data2/current/BP-1848284120-172.17.0.3-1731514082621 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T16:11:29,863 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T16:11:29,871 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6c8e2b9b{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T16:11:29,872 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@cd7e66f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T16:11:29,872 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T16:11:29,872 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4164510b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T16:11:29,873 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@34f5021e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3772cfda-71de-26d3-3229-45fcb377a4d9/hadoop.log.dir/,STOPPED} 2024-11-13T16:11:29,891 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-13T16:11:30,008 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-13T16:11:30,023 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.procedure.TestSCPWithMeta#testRecoveryAndDoubleExecutionOnRsWithMeta Thread=92 (was 11) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-6-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (308723010) connection to localhost/127.0.0.1:44161 from jenkins.hfs.1 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44161 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@43037c66 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253) app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.1@localhost:44161 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-5-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-5-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.2@localhost:44161 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (308723010) connection to localhost/127.0.0.1:44161 from jenkins.hfs.2 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-3-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-7-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-6-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SSL Certificates Store Monitor java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: nioEventLoopGroup-5-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44161 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-5-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: LeaseRenewer:jenkins@localhost:44161 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/f8f85b0bb1d8:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (308723010) connection to localhost/127.0.0.1:44161 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/f8f85b0bb1d8:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/f8f85b0bb1d8:0.logRoller java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:180) Potentially hanging thread: nioEventLoopGroup-3-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/f8f85b0bb1d8:0.Chore.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: BootstrapNodeManager java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/f8f85b0bb1d8:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44161 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (308723010) connection to localhost/127.0.0.1:44161 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: master/f8f85b0bb1d8:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Abort regionserver monitor java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44161 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: nioEventLoopGroup-2-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-7-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: nioEventLoopGroup-7-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/f8f85b0bb1d8:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-6-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 285) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=440 (was 938), ProcessCount=11 (was 11), AvailableMemoryMB=2739 (was 3482)